http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java index 75f28b6..c44e41b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java @@ -47,7 +47,7 @@ public class GenericUDAFVarianceSample extends GenericUDAFVariance { "Exactly one argument is expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE.toMetastoreTypeCategory()) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed.");
http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java index cbf3f3f..c81b0cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java @@ -67,7 +67,7 @@ public class GenericUDAFnGrams implements GenericUDAFResolver { // Validate the first parameter, which is the expression to compute over. This should be an // array of strings type, or an array of arrays of strings. PrimitiveTypeInfo pti; - if (parameters[0].getCategory() != ObjectInspector.Category.LIST) { + if (parameters[0].getCategory() != ObjectInspector.Category.LIST.toMetastoreTypeCategory()) { throw new UDFArgumentTypeException(0, "Only list type arguments are accepted but " + parameters[0].getTypeName() + " was passed as parameter 1."); @@ -98,7 +98,7 @@ public class GenericUDAFnGrams implements GenericUDAFResolver { } // Validate the second parameter, which should be an integer - if(parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE.toMetastoreTypeCategory()) { throw new UDFArgumentTypeException(1, "Only integers are accepted but " + parameters[1].getTypeName() + " was passed as parameter 2."); } @@ -116,7 +116,7 @@ public class GenericUDAFnGrams implements GenericUDAFResolver { } // Validate the third parameter, which should also be an integer - if(parameters[2].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[2].getCategory() != ObjectInspector.Category.PRIMITIVE.toMetastoreTypeCategory()) { throw new UDFArgumentTypeException(2, "Only integers are accepted but " + parameters[2].getTypeName() + " was passed as parameter 3."); } @@ -135,7 +135,7 @@ public class GenericUDAFnGrams implements GenericUDAFResolver { // If we have the optional fourth parameter, make sure it's also an integer if(parameters.length == 4) { - if(parameters[3].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[3].getCategory() != ObjectInspector.Category.PRIMITIVE.toMetastoreTypeCategory()) { throw new UDFArgumentTypeException(3, "Only integers are accepted but " + parameters[3].getTypeName() + " was passed as parameter 4."); } http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java index c91865b..76ec346 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java @@ -253,7 +253,7 @@ public final class GenericUDFUtils { TypeInfo returnType = null; for (ExprNodeDesc node : children) { TypeInfo ti = node.getTypeInfo(); - if (ti.getCategory() == Category.PRIMITIVE + if (ti.getCategory() == Category.PRIMITIVE.toMetastoreTypeCategory() && ((PrimitiveTypeInfo)ti).getPrimitiveCategory() == PrimitiveCategory.VOID) { continue; } http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java index 83cdb2d..4523c87 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java @@ -65,7 +65,7 @@ public class TestVectorSerDeRow extends TestCase { private void verifyRead( DeserializeRead deserializeRead, TypeInfo typeInfo, Object expectedObject) throws IOException { - if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE.toMetastoreTypeCategory()) { VectorVerifyFast.verifyDeserializeRead(deserializeRead, typeInfo, expectedObject); } else { Object complexFieldObj = VectorVerifyFast.deserializeReadComplexType(deserializeRead, typeInfo); http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java index 3f99328..b546289 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java @@ -435,7 +435,7 @@ public class VectorRandomRowSource { } typeInfos[c] = typeInfo; - final Category category = typeInfo.getCategory(); + final Category category = Category.fromMetastoreTypeCategory(typeInfo.getCategory()); categories[c] = category; ObjectInspector objectInspector = getObjectInspector(typeInfo); switch (category) { @@ -658,7 +658,7 @@ public class VectorRandomRowSource { elementObjectInspector); boolean isStringFamily = false; PrimitiveCategory primitiveCategory = null; - if (elementTypeInfo.getCategory() == Category.PRIMITIVE) { + if (elementTypeInfo.getCategory() == Category.PRIMITIVE.toMetastoreTypeCategory()) { primitiveCategory = ((PrimitiveTypeInfo) elementTypeInfo).getPrimitiveCategory(); if (primitiveCategory == PrimitiveCategory.STRING || primitiveCategory == PrimitiveCategory.BINARY || http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java index 10ed6d7..11a3a52 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java @@ -176,7 +176,7 @@ public class CheckFastRowHashMap extends CheckFastHashTable { private static void verifyRead(LazyBinaryDeserializeRead lazyBinaryDeserializeRead, TypeInfo typeInfo, Object expectedObject) throws IOException { - if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE.toMetastoreTypeCategory()) { VerifyFastRow.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfo, expectedObject); } else { final Object complexFieldObj = http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/ql/src/test/results/clientnegative/avro_decimal.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/avro_decimal.q.out b/ql/src/test/results/clientnegative/avro_decimal.q.out index 9d00d6e..947be05 100644 --- a/ql/src/test/results/clientnegative/avro_decimal.q.out +++ b/ql/src/test/results/clientnegative/avro_decimal.q.out @@ -19,4 +19,4 @@ TBLPROPERTIES ( PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@avro_dec -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.avro.AvroSerdeException Invalid precision or scale for decimal type) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.avro.AvroSerdeException Decimal precision out of allowed range [1,38]) http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/pom.xml ---------------------------------------------------------------------- diff --git a/serde/pom.xml b/serde/pom.xml index 0247c32..b39c6b7 100644 --- a/serde/pom.xml +++ b/serde/pom.xml @@ -49,6 +49,11 @@ <artifactId>hive-shims</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-standalone-metastore</artifactId> + <version>${project.version}</version> + </dependency> <!-- inter-project --> <dependency> <groupId>com.google.code.findbugs</groupId> http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java index 085835f..1846fba 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Properties; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java index a07cf09..54a2fd0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; /** @@ -50,7 +51,11 @@ public class AvroObjectInspectorGenerator { verifySchemaIsARecord(schema); this.columnNames = AvroObjectInspectorGenerator.generateColumnNames(schema); - this.columnTypes = SchemaToTypeInfo.generateColumnTypes(schema); + try { + this.columnTypes = SchemaToHiveTypeInfo.getInstance().generateColumnTypes(schema); + } catch (Exception e) { + throw new AvroSerdeException(e.getMessage()); + } this.columnComments = AvroObjectInspectorGenerator.generateColumnComments(schema); assert columnNames.size() == columnTypes.size(); this.oi = createObjectInspector(); @@ -139,7 +144,7 @@ public class AvroObjectInspectorGenerator { } private boolean supportedCategories(TypeInfo ti) { - final ObjectInspector.Category c = ti.getCategory(); + final Category c = Category.fromMetastoreTypeCategory(ti.getCategory()); return c.equals(ObjectInspector.Category.PRIMITIVE) || c.equals(ObjectInspector.Category.MAP) || c.equals(ObjectInspector.Category.LIST) || http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java index 83e5d68..e2043f6 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java @@ -165,7 +165,11 @@ class AvroSerializer { }; private Object serializeEnum(TypeInfo typeInfo, PrimitiveObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { - return enums.retrieve(schema).retrieve(serializePrimitive(typeInfo, fieldOI, structFieldData, schema)); + try { + return enums.retrieve(schema).retrieve(serializePrimitive(typeInfo, fieldOI, structFieldData, schema)); + } catch (Exception e) { + throw new AvroSerdeException(e); + } } private Object serializeStruct(StructTypeInfo typeInfo, StructObjectInspector ssoi, Object o, Schema schema) throws AvroSerdeException { http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java deleted file mode 100644 index 2d52020..0000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Cache for objects whose creation only depends on some other set of objects and therefore can be - * used against other equivalent versions of those objects. Essentially memoizes instance creation. - * - * @param <SeedObject> Object that determines the instance. The cache uses this object as a key for - * its hash which is why it is imperative to have appropriate equals and hashcode - * implementation for this object for the cache to work properly - * @param <Instance> Instance that will be created from SeedObject. - */ -public abstract class InstanceCache<SeedObject, Instance> { - private static final Logger LOG = LoggerFactory.getLogger(InstanceCache.class); - Map<SeedObject, Instance> cache = new HashMap<SeedObject, Instance>(); - - public InstanceCache() {} - - /** - * Retrieve (or create if it doesn't exist) the correct Instance for this - * SeedObject - */ - public Instance retrieve(SeedObject hv) throws AvroSerdeException { - return retrieve(hv, null); - } - - /** - * Retrieve (or create if it doesn't exist) the correct Instance for this - * SeedObject using 'seenSchemas' to resolve circular references - */ - public synchronized Instance retrieve(SeedObject hv, - Set<SeedObject> seenSchemas) throws AvroSerdeException { - if(LOG.isDebugEnabled()) LOG.debug("Checking for hv: " + hv.toString()); - - if(cache.containsKey(hv)) { - if(LOG.isDebugEnabled()) LOG.debug("Returning cache result."); - return cache.get(hv); - } - - if(LOG.isDebugEnabled()) LOG.debug("Creating new instance and storing in cache"); - - Instance instance = makeInstance(hv, seenSchemas); - cache.put(hv, instance); - return instance; - } - - protected abstract Instance makeInstance(SeedObject hv, - Set<SeedObject> seenSchemas) throws AvroSerdeException; -} http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java deleted file mode 100644 index 65f104d..0000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import org.apache.avro.Schema; - -class SchemaResolutionProblem { - static final String sentinelString = "{\n" + - " \"namespace\": \"org.apache.hadoop.hive\",\n" + - " \"name\": \"CannotDetermineSchemaSentinel\",\n" + - " \"type\": \"record\",\n" + - " \"fields\": [\n" + - " {\n" + - " \"name\":\"ERROR_ERROR_ERROR_ERROR_ERROR_ERROR_ERROR\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"Cannot_determine_schema\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"check\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"schema\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"url\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"and\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"literal\",\n" + - " \"type\":\"string\"\n" + - " }\n" + - " ]\n" + - "}"; - public final static Schema SIGNAL_BAD_SCHEMA = AvroSerdeUtils.getSchemaFor(sentinelString); -} http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToHiveTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToHiveTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToHiveTypeInfo.java new file mode 100644 index 0000000..d82f511 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToHiveTypeInfo.java @@ -0,0 +1,18 @@ +package org.apache.hadoop.hive.serde2.avro; + +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public class SchemaToHiveTypeInfo extends SchemaToTypeInfo { + private static final SchemaToHiveTypeInfo instance = new SchemaToHiveTypeInfo(); + + private SchemaToHiveTypeInfo() { + //use getInstance to get this object. The base class uses cache to reuse + //Types when available + super(TypeInfoFactory.getInstance()); + } + + public static final SchemaToHiveTypeInfo getInstance() { + return instance; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java deleted file mode 100644 index 35d83bd..0000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import static org.apache.avro.Schema.Type.BOOLEAN; -import static org.apache.avro.Schema.Type.BYTES; -import static org.apache.avro.Schema.Type.DOUBLE; -import static org.apache.avro.Schema.Type.FIXED; -import static org.apache.avro.Schema.Type.FLOAT; -import static org.apache.avro.Schema.Type.INT; -import static org.apache.avro.Schema.Type.LONG; -import static org.apache.avro.Schema.Type.NULL; -import static org.apache.avro.Schema.Type.STRING; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Hashtable; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.avro.Schema; -import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; - -/** - * Convert an Avro Schema to a Hive TypeInfo - */ -class SchemaToTypeInfo { - // Conversion of Avro primitive types to Hive primitive types - // Avro Hive - // Null - // boolean boolean check - // int int check - // long bigint check - // float double check - // double double check - // bytes binary check - // fixed binary check - // string string check - // tinyint - // smallint - - // Map of Avro's primitive types to Hives (for those that are supported by both) - private static final Map<Schema.Type, TypeInfo> primitiveTypeToTypeInfo = initTypeMap(); - private static Map<Schema.Type, TypeInfo> initTypeMap() { - Map<Schema.Type, TypeInfo> theMap = new Hashtable<Schema.Type, TypeInfo>(); - theMap.put(NULL, TypeInfoFactory.getPrimitiveTypeInfo("void")); - theMap.put(BOOLEAN, TypeInfoFactory.getPrimitiveTypeInfo("boolean")); - theMap.put(INT, TypeInfoFactory.getPrimitiveTypeInfo("int")); - theMap.put(LONG, TypeInfoFactory.getPrimitiveTypeInfo("bigint")); - theMap.put(FLOAT, TypeInfoFactory.getPrimitiveTypeInfo("float")); - theMap.put(DOUBLE, TypeInfoFactory.getPrimitiveTypeInfo("double")); - theMap.put(BYTES, TypeInfoFactory.getPrimitiveTypeInfo("binary")); - theMap.put(FIXED, TypeInfoFactory.getPrimitiveTypeInfo("binary")); - theMap.put(STRING, TypeInfoFactory.getPrimitiveTypeInfo("string")); - return Collections.unmodifiableMap(theMap); - } - - /** - * Generate a list of of TypeInfos from an Avro schema. This method is - * currently public due to some weirdness in deserializing unions, but - * will be made private once that is resolved. - * @param schema Schema to generate field types for - * @return List of TypeInfos, each element of which is a TypeInfo derived - * from the schema. - * @throws AvroSerdeException for problems during conversion. - */ - public static List<TypeInfo> generateColumnTypes(Schema schema) throws AvroSerdeException { - return generateColumnTypes (schema, null); - } - - /** - * Generate a list of of TypeInfos from an Avro schema. This method is - * currently public due to some weirdness in deserializing unions, but - * will be made private once that is resolved. - * @param schema Schema to generate field types for - * @param seenSchemas stores schemas processed in the parsing done so far, - * helping to resolve circular references in the schema - * @return List of TypeInfos, each element of which is a TypeInfo derived - * from the schema. - * @throws AvroSerdeException for problems during conversion. - */ - public static List<TypeInfo> generateColumnTypes(Schema schema, - Set<Schema> seenSchemas) throws AvroSerdeException { - List<Schema.Field> fields = schema.getFields(); - - List<TypeInfo> types = new ArrayList<TypeInfo>(fields.size()); - - for (Schema.Field field : fields) { - types.add(generateTypeInfo(field.schema(), seenSchemas)); - } - - return types; - } - - static InstanceCache<Schema, TypeInfo> typeInfoCache = new InstanceCache<Schema, TypeInfo>() { - @Override - protected TypeInfo makeInstance(Schema s, - Set<Schema> seenSchemas) - throws AvroSerdeException { - return generateTypeInfoWorker(s, seenSchemas); - } - }; - /** - * Convert an Avro Schema into an equivalent Hive TypeInfo. - * @param schema to record. Must be of record type. - * @param seenSchemas stores schemas processed in the parsing done so far, - * helping to resolve circular references in the schema - * @return TypeInfo matching the Avro schema - * @throws AvroSerdeException for any problems during conversion. - */ - public static TypeInfo generateTypeInfo(Schema schema, - Set<Schema> seenSchemas) throws AvroSerdeException { - // For bytes type, it can be mapped to decimal. - Schema.Type type = schema.getType(); - if (type == BYTES && AvroSerDe.DECIMAL_TYPE_NAME - .equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - int precision = 0; - int scale = 0; - try { - precision = schema.getJsonProp(AvroSerDe.AVRO_PROP_PRECISION).getIntValue(); - scale = schema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).getIntValue(); - } catch (Exception ex) { - throw new AvroSerdeException("Failed to obtain scale value from file schema: " + schema, ex); - } - - try { - HiveDecimalUtils.validateParameter(precision, scale); - } catch (Exception ex) { - throw new AvroSerdeException("Invalid precision or scale for decimal type", ex); - } - - return TypeInfoFactory.getDecimalTypeInfo(precision, scale); - } - - if (type == STRING && - AvroSerDe.CHAR_TYPE_NAME.equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - int maxLength = 0; - try { - maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt(); - } catch (Exception ex) { - throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex); - } - return TypeInfoFactory.getCharTypeInfo(maxLength); - } - - if (type == STRING && AvroSerDe.VARCHAR_TYPE_NAME - .equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - int maxLength = 0; - try { - maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt(); - } catch (Exception ex) { - throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex); - } - return TypeInfoFactory.getVarcharTypeInfo(maxLength); - } - - if (type == INT && - AvroSerDe.DATE_TYPE_NAME.equals(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - return TypeInfoFactory.dateTypeInfo; - } - - if (type == LONG && - AvroSerDe.TIMESTAMP_TYPE_NAME.equals(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - return TypeInfoFactory.timestampTypeInfo; - } - - return typeInfoCache.retrieve(schema, seenSchemas); - } - - private static TypeInfo generateTypeInfoWorker(Schema schema, - Set<Schema> seenSchemas) throws AvroSerdeException { - // Avro requires NULLable types to be defined as unions of some type T - // and NULL. This is annoying and we're going to hide it from the user. - if(AvroSerdeUtils.isNullableType(schema)) { - return generateTypeInfo( - AvroSerdeUtils.getOtherTypeFromNullableType(schema), seenSchemas); - } - - Schema.Type type = schema.getType(); - if(primitiveTypeToTypeInfo.containsKey(type)) { - return primitiveTypeToTypeInfo.get(type); - } - - switch(type) { - case RECORD: return generateRecordTypeInfo(schema, seenSchemas); - case MAP: return generateMapTypeInfo(schema, seenSchemas); - case ARRAY: return generateArrayTypeInfo(schema, seenSchemas); - case UNION: return generateUnionTypeInfo(schema, seenSchemas); - case ENUM: return generateEnumTypeInfo(schema); - default: throw new AvroSerdeException("Do not yet support: " + schema); - } - } - - private static TypeInfo generateRecordTypeInfo(Schema schema, - Set<Schema> seenSchemas) throws AvroSerdeException { - assert schema.getType().equals(Schema.Type.RECORD); - - if (seenSchemas == null) { - seenSchemas = Collections.newSetFromMap(new IdentityHashMap<Schema, Boolean>()); - } else if (seenSchemas.contains(schema)) { - throw new AvroSerdeException( - "Recursive schemas are not supported. Recursive schema was " + schema - .getFullName()); - } - seenSchemas.add(schema); - - List<Schema.Field> fields = schema.getFields(); - List<String> fieldNames = new ArrayList<String>(fields.size()); - List<TypeInfo> typeInfos = new ArrayList<TypeInfo>(fields.size()); - - for(int i = 0; i < fields.size(); i++) { - fieldNames.add(i, fields.get(i).name()); - typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas)); - } - - return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); - } - - /** - * Generate a TypeInfo for an Avro Map. This is made slightly simpler in that - * Avro only allows maps with strings for keys. - */ - private static TypeInfo generateMapTypeInfo(Schema schema, - Set<Schema> seenSchemas) throws AvroSerdeException { - assert schema.getType().equals(Schema.Type.MAP); - Schema valueType = schema.getValueType(); - TypeInfo ti = generateTypeInfo(valueType, seenSchemas); - - return TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("string"), ti); - } - - private static TypeInfo generateArrayTypeInfo(Schema schema, - Set<Schema> seenSchemas) throws AvroSerdeException { - assert schema.getType().equals(Schema.Type.ARRAY); - Schema itemsType = schema.getElementType(); - TypeInfo itemsTypeInfo = generateTypeInfo(itemsType, seenSchemas); - - return TypeInfoFactory.getListTypeInfo(itemsTypeInfo); - } - - private static TypeInfo generateUnionTypeInfo(Schema schema, - Set<Schema> seenSchemas) throws AvroSerdeException { - assert schema.getType().equals(Schema.Type.UNION); - List<Schema> types = schema.getTypes(); - - - List<TypeInfo> typeInfos = new ArrayList<TypeInfo>(types.size()); - - for(Schema type : types) { - typeInfos.add(generateTypeInfo(type, seenSchemas)); - } - - return TypeInfoFactory.getUnionTypeInfo(typeInfos); - } - - // Hive doesn't have an Enum type, so we're going to treat them as Strings. - // During the deserialize/serialize stage we'll check for enumness and - // convert as such. - private static TypeInfo generateEnumTypeInfo(Schema schema) { - assert schema.getType().equals(Schema.Type.ENUM); - - return TypeInfoFactory.getPrimitiveTypeInfo("string"); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java deleted file mode 100644 index 4f8b05f..0000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import org.apache.avro.Schema; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.node.JsonNodeFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Convert Hive TypeInfo to an Avro Schema - */ -public class TypeInfoToSchema { - - private long recordCounter = 0; - - /** - * Converts Hive schema to avro schema - * - * @param columnNames Names of the hive columns - * @param columnTypes Hive Column types - * @param namespace Namespace of Avro schema - * @param name Avro schema name - * @param doc Avro schema doc - * @return Avro Schema - */ - public Schema convert(List<String> columnNames, List<TypeInfo> columnTypes, - List<String> columnComments, String namespace, String name, String doc) { - - List<Schema.Field> fields = new ArrayList<Schema.Field>(); - for (int i = 0; i < columnNames.size(); ++i) { - final String comment = columnComments.size() > i ? columnComments.get(i) : null; - final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), - comment); - fields.addAll(getFields(avroField)); - } - - if (name == null || name.isEmpty()) { - name = "baseRecord"; - } - - Schema avroSchema = Schema.createRecord(name, doc, namespace, false); - avroSchema.setFields(fields); - return avroSchema; - } - - private Schema.Field createAvroField(String name, TypeInfo typeInfo, String comment) { - return new Schema.Field(name, createAvroSchema(typeInfo), comment, null); - } - - private Schema createAvroSchema(TypeInfo typeInfo) { - Schema schema = null; - switch (typeInfo.getCategory()) { - case PRIMITIVE: - schema = createAvroPrimitive(typeInfo); - break; - case LIST: - schema = createAvroArray(typeInfo); - break; - case MAP: - schema = createAvroMap(typeInfo); - break; - case STRUCT: - schema = createAvroRecord(typeInfo); - break; - case UNION: - schema = createAvroUnion(typeInfo); - break; - } - - return wrapInUnionWithNull(schema); - } - - private Schema createAvroPrimitive(TypeInfo typeInfo) { - PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; - Schema schema; - switch (primitiveTypeInfo.getPrimitiveCategory()) { - case STRING: - schema = Schema.create(Schema.Type.STRING); - break; - case CHAR: - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"" + AvroSerDe.AVRO_STRING_TYPE_NAME + "\"," + - "\"logicalType\":\"" + AvroSerDe.CHAR_TYPE_NAME + "\"," + - "\"maxLength\":" + ((CharTypeInfo) typeInfo).getLength() + "}"); - break; - case VARCHAR: - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"" + AvroSerDe.AVRO_STRING_TYPE_NAME + "\"," + - "\"logicalType\":\"" + AvroSerDe.VARCHAR_TYPE_NAME + "\"," + - "\"maxLength\":" + ((VarcharTypeInfo) typeInfo).getLength() + "}"); - break; - case BINARY: - schema = Schema.create(Schema.Type.BYTES); - break; - case BYTE: - schema = Schema.create(Schema.Type.INT); - break; - case SHORT: - schema = Schema.create(Schema.Type.INT); - break; - case INT: - schema = Schema.create(Schema.Type.INT); - break; - case LONG: - schema = Schema.create(Schema.Type.LONG); - break; - case FLOAT: - schema = Schema.create(Schema.Type.FLOAT); - break; - case DOUBLE: - schema = Schema.create(Schema.Type.DOUBLE); - break; - case BOOLEAN: - schema = Schema.create(Schema.Type.BOOLEAN); - break; - case DECIMAL: - DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; - String precision = String.valueOf(decimalTypeInfo.precision()); - String scale = String.valueOf(decimalTypeInfo.scale()); - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"bytes\"," + - "\"logicalType\":\"decimal\"," + - "\"precision\":" + precision + "," + - "\"scale\":" + scale + "}"); - break; - case DATE: - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"" + AvroSerDe.AVRO_INT_TYPE_NAME + "\"," + - "\"logicalType\":\"" + AvroSerDe.DATE_TYPE_NAME + "\"}"); - break; - case TIMESTAMP: - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"" + AvroSerDe.AVRO_LONG_TYPE_NAME + "\"," + - "\"logicalType\":\"" + AvroSerDe.TIMESTAMP_TYPE_NAME + "\"}"); - break; - case VOID: - schema = Schema.create(Schema.Type.NULL); - break; - default: - throw new UnsupportedOperationException(typeInfo + " is not supported."); - } - return schema; - } - - private Schema createAvroUnion(TypeInfo typeInfo) { - List<Schema> childSchemas = new ArrayList<Schema>(); - for (TypeInfo childTypeInfo : ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos()) { - final Schema childSchema = createAvroSchema(childTypeInfo); - if (childSchema.getType() == Schema.Type.UNION) { - childSchemas.addAll(childSchema.getTypes()); - } else { - childSchemas.add(childSchema); - } - } - - return Schema.createUnion(removeDuplicateNullSchemas(childSchemas)); - } - - private Schema createAvroRecord(TypeInfo typeInfo) { - List<Schema.Field> childFields = new ArrayList<Schema.Field>(); - - final List<String> allStructFieldNames = - ((StructTypeInfo) typeInfo).getAllStructFieldNames(); - final List<TypeInfo> allStructFieldTypeInfos = - ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); - if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { - throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + - "name and column type differs. names = " + allStructFieldNames + ", types = " + - allStructFieldTypeInfos); - } - - for (int i = 0; i < allStructFieldNames.size(); ++i) { - final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); - final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), - childTypeInfo, childTypeInfo.toString()); - final List<Schema.Field> grandChildFields = getFields(grandChildSchemaField); - childFields.addAll(grandChildFields); - } - - Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), - null, false); - ++recordCounter; - recordSchema.setFields(childFields); - return recordSchema; - } - - private Schema createAvroMap(TypeInfo typeInfo) { - TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); - if (((PrimitiveTypeInfo) keyTypeInfo).getPrimitiveCategory() - != PrimitiveObjectInspector.PrimitiveCategory.STRING) { - throw new UnsupportedOperationException("Key of Map can only be a String"); - } - - TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); - Schema valueSchema = createAvroSchema(valueTypeInfo); - - return Schema.createMap(valueSchema); - } - - private Schema createAvroArray(TypeInfo typeInfo) { - ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; - Schema listSchema = createAvroSchema(listTypeInfo.getListElementTypeInfo()); - return Schema.createArray(listSchema); - } - - private List<Schema.Field> getFields(Schema.Field schemaField) { - List<Schema.Field> fields = new ArrayList<Schema.Field>(); - - JsonNode nullDefault = JsonNodeFactory.instance.nullNode(); - if (schemaField.schema().getType() == Schema.Type.RECORD) { - for (Schema.Field field : schemaField.schema().getFields()) { - fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault)); - } - } else { - fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(), - nullDefault)); - } - - return fields; - } - - private Schema wrapInUnionWithNull(Schema schema) { - Schema wrappedSchema = schema; - switch (schema.getType()) { - case NULL: - break; - case UNION: - List<Schema> existingSchemas = removeDuplicateNullSchemas(schema.getTypes()); - wrappedSchema = Schema.createUnion(existingSchemas); - break; - default: - wrappedSchema = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema)); - } - - return wrappedSchema; - } - - private List<Schema> removeDuplicateNullSchemas(List<Schema> childSchemas) { - List<Schema> prunedSchemas = new ArrayList<Schema>(); - boolean isNullPresent = false; - for (Schema schema : childSchemas) { - if (schema.getType() == Schema.Type.NULL) { - isNullPresent = true; - } else { - prunedSchemas.add(schema); - } - } - if (isNullPresent) { - prunedSchemas.add(0, Schema.create(Schema.Type.NULL)); - } - - return prunedSchemas; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java index 461043d..29751b0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java @@ -590,7 +590,7 @@ public final class BinarySortableDeserializeRead extends DeserializeRead { private Field createField(TypeInfo typeInfo) { final Field field = new Field(); - final Category category = typeInfo.getCategory(); + final Category category = Category.fromMetastoreTypeCategory(typeInfo.getCategory()); field.category = category; field.typeInfo = typeInfo; http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java index 197031d..9adf29f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java @@ -161,7 +161,7 @@ public abstract class DeserializeRead { primitiveCategories = new PrimitiveCategory[count]; for (int i = 0; i < count; i++) { TypeInfo typeInfo = typeInfos[i]; - Category category = typeInfo.getCategory(); + Category category = Category.fromMetastoreTypeCategory(typeInfo.getCategory()); categories[i] = category; if (category == Category.PRIMITIVE) { PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index 883ba88..e3ade95 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -325,7 +325,7 @@ public final class LazyFactory { public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, int separatorIndex, LazyObjectInspectorParameters lazyParams, ObjectInspectorOptions option) throws SerDeException { - ObjectInspector.Category c = typeInfo.getCategory(); + ObjectInspector.Category c = Category.fromMetastoreTypeCategory(typeInfo.getCategory()); switch (c) { case PRIMITIVE: return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector( http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index fe0ee48..146954b 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@ -89,7 +89,7 @@ public final class LazySimpleDeserializeRead extends DeserializeRead { public ComplexTypeHelper complexTypeHelper; public Field(TypeInfo typeInfo, DataTypePhysicalVariation dataTypePhysicalVariation) { - Category category = typeInfo.getCategory(); + Category category = Category.fromMetastoreTypeCategory(typeInfo.getCategory()); if (category == Category.PRIMITIVE) { isPrimitive = true; primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java index 340f322..a23753f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java @@ -112,7 +112,7 @@ public final class LazyBinaryDeserializeRead extends DeserializeRead { private Field createField(TypeInfo typeInfo) { final Field field = new Field(); - final Category category = typeInfo.getCategory(); + final Category category = Category.fromMetastoreTypeCategory(typeInfo.getCategory()); field.category = category; field.typeInfo = typeInfo; switch (category) { http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java index 99b565d..2baa1fa 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java @@ -1,25 +1,27 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.serde2.objectinspector; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.typeinfo.MetastoreTypeCategory; /** * ObjectInspector helps us to look into the internal structure of a complex @@ -47,7 +49,60 @@ public interface ObjectInspector extends Cloneable { * */ public static enum Category { - PRIMITIVE, LIST, MAP, STRUCT, UNION + PRIMITIVE, LIST, MAP, STRUCT, UNION; + + /** + * This utility method maps the MetastoreTypeCategory enum to Category enum. + * @param metastoreTypeCategory + * @return Category enum equivalent from MetastoreTypeCategory + */ + public static Category fromMetastoreTypeCategory(MetastoreTypeCategory metastoreTypeCategory) { + switch (metastoreTypeCategory) { + case PRIMITIVE: + return Category.PRIMITIVE; + case LIST: + return Category.LIST; + case MAP: + return Category.MAP; + case STRUCT: + return Category.STRUCT; + case UNION: + return Category.UNION; + default: + throw new RuntimeException("Unsupported metastore type category " + metastoreTypeCategory); + } + } + + /** + * returns the MetastoreTypeCategory enum mapping of this enum value + * @return + */ + public MetastoreTypeCategory toMetastoreTypeCategory() { + switch (this) { + case PRIMITIVE: + return MetastoreTypeCategory.PRIMITIVE; + case LIST: + return MetastoreTypeCategory.LIST; + case MAP: + return MetastoreTypeCategory.MAP; + case STRUCT: + return MetastoreTypeCategory.STRUCT; + case UNION: + return MetastoreTypeCategory.UNION; + default: + throw new RuntimeException( + "Unsupported mapping to metastore type category " + this.toString()); + } + } + + /** + * Util method for mapping Category enum to MetastoreTypeCategory enum + * @param metastoreTypeCategory + * @return + */ + public boolean equals(MetastoreTypeCategory metastoreTypeCategory) { + return this.toMetastoreTypeCategory().equals(metastoreTypeCategory); + } }; /** http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java index 820fb4e..4ffe50f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java @@ -67,4 +67,9 @@ public abstract class BaseCharTypeInfo extends PrimitiveTypeInfo { // type name should already be set by subclass return; } + + @Override + public Object[] getParameters() { + return new Object[] { length }; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java index 2e76df5..c5cb32b 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java @@ -127,4 +127,9 @@ public class DecimalTypeInfo extends PrimitiveTypeInfo { this.scale = scale; } + @Override + public Object[] getParameters() { + return new Object[] { precision, scale }; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java deleted file mode 100644 index c632bff..0000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * A List Type has homogeneous elements. All elements of the List has the same - * TypeInfo which is returned by getListElementTypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class ListTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - private TypeInfo listElementTypeInfo; - - /** - * For java serialization use only. - */ - public ListTypeInfo() { - } - - @Override - public String getTypeName() { - return org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "<" - + listElementTypeInfo.getTypeName() + ">"; - } - - /** - * For java serialization use only. - */ - public void setListElementTypeInfo(TypeInfo listElementTypeInfo) { - this.listElementTypeInfo = listElementTypeInfo; - } - - /** - * For TypeInfoFactory use only. - */ - ListTypeInfo(TypeInfo elementTypeInfo) { - listElementTypeInfo = elementTypeInfo; - } - - @Override - public Category getCategory() { - return Category.LIST; - } - - public TypeInfo getListElementTypeInfo() { - return listElementTypeInfo; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof ListTypeInfo)) { - return false; - } - return getListElementTypeInfo().equals( - ((ListTypeInfo) other).getListElementTypeInfo()); - } - - @Override - public int hashCode() { - return listElementTypeInfo.hashCode(); - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java deleted file mode 100644 index 1344c90..0000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * A Map Type has homogeneous keys and homogeneous values. All keys of the Map - * have the same TypeInfo, which is returned by getMapKeyTypeInfo(); and all - * values of the Map has the same TypeInfo, which is returned by - * getMapValueTypeInfo(). - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class MapTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private TypeInfo mapKeyTypeInfo; - private TypeInfo mapValueTypeInfo; - - /** - * For java serialization use only. - */ - public MapTypeInfo() { - } - - @Override - public String getTypeName() { - return org.apache.hadoop.hive.serde.serdeConstants.MAP_TYPE_NAME + "<" - + mapKeyTypeInfo.getTypeName() + "," + mapValueTypeInfo.getTypeName() - + ">"; - } - - /** - * For java serialization use only. - */ - public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) { - this.mapKeyTypeInfo = mapKeyTypeInfo; - } - - /** - * For java serialization use only. - */ - public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) { - this.mapValueTypeInfo = mapValueTypeInfo; - } - - // For TypeInfoFactory use only - MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { - mapKeyTypeInfo = keyTypeInfo; - mapValueTypeInfo = valueTypeInfo; - } - - @Override - public Category getCategory() { - return Category.MAP; - } - - public TypeInfo getMapKeyTypeInfo() { - return mapKeyTypeInfo; - } - - public TypeInfo getMapValueTypeInfo() { - return mapValueTypeInfo; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof MapTypeInfo)) { - return false; - } - MapTypeInfo o = (MapTypeInfo) other; - return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) - && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); - } - - @Override - public int hashCode() { - return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode(); - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java index 97af49a..d55e0c7 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java @@ -22,6 +22,7 @@ import java.io.Serializable; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; @@ -36,16 +37,14 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn */ @InterfaceAudience.Public @InterfaceStability.Stable -public class PrimitiveTypeInfo extends TypeInfo implements Serializable { +public class PrimitiveTypeInfo extends MetastorePrimitiveTypeInfo implements Serializable { private static final long serialVersionUID = 1L; - // Base name (varchar vs fully qualified name such as varchar(200)). - protected String typeName; - /** * For java serialization use only. */ public PrimitiveTypeInfo() { + super(); } /** @@ -59,8 +58,8 @@ public class PrimitiveTypeInfo extends TypeInfo implements Serializable { * Returns the category of this TypeInfo. */ @Override - public Category getCategory() { - return Category.PRIMITIVE; + public MetastoreTypeCategory getCategory() { + return MetastoreTypeCategory.PRIMITIVE; } public PrimitiveCategory getPrimitiveCategory() { @@ -75,44 +74,7 @@ public class PrimitiveTypeInfo extends TypeInfo implements Serializable { return getPrimitiveTypeEntry().primitiveJavaClass; } - // The following 2 methods are for java serialization use only. - public void setTypeName(String typeName) { - this.typeName = typeName; - } - - @Override - public String getTypeName() { - return typeName; - } - public PrimitiveTypeEntry getPrimitiveTypeEntry() { return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName); } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - - PrimitiveTypeInfo pti = (PrimitiveTypeInfo) other; - - return this.typeName.equals(pti.typeName); - } - - /** - * Generate the hashCode for this TypeInfo. - */ - @Override - public int hashCode() { - return typeName.hashCode(); - } - - @Override - public String toString() { - return typeName; - } } http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java deleted file mode 100644 index 4caedb0..0000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * StructTypeInfo represents the TypeInfo of a struct. A struct contains one or - * more fields each of which has a unique name and its own TypeInfo. Different - * fields can have the same or different TypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class StructTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private ArrayList<String> allStructFieldNames; - private ArrayList<TypeInfo> allStructFieldTypeInfos; - - /** - * For java serialization use only. - */ - public StructTypeInfo() { - } - - @Override - public String getTypeName() { - StringBuilder sb = new StringBuilder(); - sb.append(serdeConstants.STRUCT_TYPE_NAME + "<"); - for (int i = 0; i < allStructFieldNames.size(); i++) { - if (i > 0) { - sb.append(","); - } - sb.append(allStructFieldNames.get(i)); - sb.append(":"); - sb.append(allStructFieldTypeInfos.get(i).getTypeName()); - } - sb.append(">"); - return sb.toString(); - } - - /** - * For java serialization use only. - */ - public void setAllStructFieldNames(ArrayList<String> allStructFieldNames) { - this.allStructFieldNames = allStructFieldNames; - } - - /** - * For java serialization use only. - */ - public void setAllStructFieldTypeInfos( - ArrayList<TypeInfo> allStructFieldTypeInfos) { - this.allStructFieldTypeInfos = allStructFieldTypeInfos; - } - - /** - * For TypeInfoFactory use only. - */ - StructTypeInfo(List<String> names, List<TypeInfo> typeInfos) { - allStructFieldNames = new ArrayList<String>(names); - allStructFieldTypeInfos = new ArrayList<TypeInfo>(typeInfos); - } - - @Override - public Category getCategory() { - return Category.STRUCT; - } - - public ArrayList<String> getAllStructFieldNames() { - return allStructFieldNames; - } - - public ArrayList<TypeInfo> getAllStructFieldTypeInfos() { - return allStructFieldTypeInfos; - } - - public TypeInfo getStructFieldTypeInfo(String field) { - String fieldLowerCase = field.toLowerCase(); - for (int i = 0; i < allStructFieldNames.size(); i++) { - if (fieldLowerCase.equalsIgnoreCase(allStructFieldNames.get(i))) { - return allStructFieldTypeInfos.get(i); - } - } - throw new RuntimeException("cannot find field " + field - + "(lowercase form: " + fieldLowerCase + ") in " + allStructFieldNames); - // return null; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof StructTypeInfo)) { - return false; - } - StructTypeInfo o = (StructTypeInfo) other; - Iterator<String> namesIterator = getAllStructFieldNames().iterator(); - Iterator<String> otherNamesIterator = o.getAllStructFieldNames().iterator(); - - // Compare the field names using ignore-case semantics - while (namesIterator.hasNext() && otherNamesIterator.hasNext()) { - if (!namesIterator.next().equalsIgnoreCase(otherNamesIterator.next())) { - return false; - } - } - - // Different number of field names - if (namesIterator.hasNext() || otherNamesIterator.hasNext()) { - return false; - } - - // Compare the field types - return o.getAllStructFieldTypeInfos().equals(getAllStructFieldTypeInfos()); - } - - @Override - public int hashCode() { - return allStructFieldNames.hashCode() ^ allStructFieldTypeInfos.hashCode(); - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java index 6f9eeea..8b94208 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java @@ -101,4 +101,9 @@ public class TimestampLocalTZTypeInfo extends PrimitiveTypeInfo { this.timeZone = timeZone; } + @Override + public Object[] getParameters() { + return new Object[] { timeZone }; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java deleted file mode 100644 index 75e0973..0000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * Stores information about a type. Always use the TypeInfoFactory to create new - * TypeInfo objects. - * - * We support 8 categories of types: - * 1. Primitive objects (String, Number, etc) - * 2. List objects (a list of objects of a single type) - * 3. Map objects (a map from objects of one type to objects of another type) - * 4. Struct objects (a list of fields with names and their own types) - * 5. Union objects - * 6. Decimal objects - * 7. Char objects - * 8. Varchar objects - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public abstract class TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - protected TypeInfo() { - } - - /** - * The Category of this TypeInfo. Possible values are Primitive, List, Map, - * Struct and Union, which corresponds to the 5 sub-classes of TypeInfo. - */ - public abstract Category getCategory(); - - /** - * A String representation of the TypeInfo. - */ - public abstract String getTypeName(); - - /** - * String representing the qualified type name. - * Qualified types should override this method. - * @return - */ - public String getQualifiedName() { - return getTypeName(); - } - - @Override - public String toString() { - return getTypeName(); - } - - @Override - public abstract boolean equals(Object o); - - @Override - public abstract int hashCode(); - - public boolean accept(TypeInfo other) { - return this.equals(other); - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/40ee74eb/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index 77d60c5..e730e9d 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoParser.PrimitiveParts; /** * TypeInfoFactory can be used to create the TypeInfo object for any types. @@ -39,6 +40,53 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn */ public final class TypeInfoFactory { + //if this singleton looks weird, it is. It is done to support the move of TypeInfo and its sub-classes + //to standalone metastore. Standalone metastore needs access to TypeInfos without depending + //on hive-serde. In order to create these TypeInfo, it needs the TypeInfoParser and TypeInfoFactory. + //We cannot move TypeInfoFactory out of hive code base since its too disruptive. + //ITypeInfoFactory interface in standalone-metastore abstracts out the common functionality + //and metastore implements its own version of TypeInfoFactory. Hive uses its original TypeInfoFactory + //without any changes. The singleton weird-ness is needed to keep the caching behaviour when + //using this TypeInfoFactory in Deserializer and TypeInfoParsers + private static final ITypeInfoFactory instance = new ITypeInfoFactory() { + @Override + public MetastorePrimitiveTypeInfo getPrimitiveTypeInfo(String typeName, Object... parameters) { + if (serdeConstants.DECIMAL_TYPE_NAME.equals(typeName)) { + HiveDecimalUtils.validateParameter((Integer) parameters[0], (Integer) parameters[1]); + } else if (serdeConstants.CHAR_TYPE_NAME.equals(typeName)) { + BaseCharUtils.validateCharParameter((Integer) parameters[0]); + } else if (serdeConstants.VARCHAR_TYPE_NAME.equals(typeName)) { + BaseCharUtils.validateVarcharParameter((Integer)parameters[0]); + } + return TypeInfoFactory.getPrimitiveTypeInfo( + MetastoreTypeInfoUtils.getQualifiedPrimitiveTypeName(typeName, parameters)); + } + + @Override + public MapTypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { + return (MapTypeInfo) TypeInfoFactory.getMapTypeInfo(keyTypeInfo, valueTypeInfo); + } + + @Override + public ListTypeInfo getListTypeInfo(TypeInfo listElementTypeInfo) { + return (ListTypeInfo) TypeInfoFactory.getListTypeInfo(listElementTypeInfo); + } + + @Override + public UnionTypeInfo getUnionTypeInfo(List<TypeInfo> typeInfos) { + return (UnionTypeInfo) TypeInfoFactory.getUnionTypeInfo(typeInfos); + } + + @Override + public StructTypeInfo getStructTypeInfo(List<String> names, List<TypeInfo> typeInfos) { + return (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(names, typeInfos); + } + }; + + public static final ITypeInfoFactory getInstance() { + return instance; + } + private TypeInfoFactory() { // prevent instantiation } @@ -134,14 +182,14 @@ public final class TypeInfoFactory { * @return PrimitiveTypeInfo instance */ private static PrimitiveTypeInfo createPrimitiveTypeInfo(String fullName) { - String baseName = TypeInfoUtils.getBaseName(fullName); + String baseName = MetastoreTypeInfoUtils.getBaseName(fullName); PrimitiveTypeEntry typeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(baseName); if (null == typeEntry) { throw new RuntimeException("Unknown type " + fullName); } - TypeInfoUtils.PrimitiveParts parts = TypeInfoUtils.parsePrimitiveParts(fullName); + PrimitiveParts parts = TypeInfoUtils.parsePrimitiveParts(fullName); if (parts.typeParams == null || parts.typeParams.length < 1) { return null; }
