Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java?rev=1523463&view=auto ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java (added) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java Sun Sep 15 17:23:53 2013 @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; + +public class WritableHiveVarcharObjectInspector + extends AbstractPrimitiveWritableObjectInspector + implements SettableHiveVarcharObjectInspector { + + private static final Log LOG = LogFactory.getLog(WritableHiveVarcharObjectInspector.class); + + // no-arg ctor required for Kyro serialization + public WritableHiveVarcharObjectInspector() { + } + + public WritableHiveVarcharObjectInspector(PrimitiveTypeEntry typeEntry) { + super(typeEntry); + if (typeEntry.primitiveCategory != PrimitiveCategory.VARCHAR) { + throw new RuntimeException( + "TypeEntry of type varchar expected, got " + typeEntry.primitiveCategory); + } + } + + @Override + public HiveVarchar getPrimitiveJavaObject(Object o) { + // check input object's length, if it doesn't match + // then output a new primitive with the correct params. + if (o == null) { + return null; + } + HiveVarcharWritable writable = ((HiveVarcharWritable)o); + if (doesWritableMatchTypeParams(writable)) { + return writable.getHiveVarchar(); + } + return getPrimitiveWithParams(writable); + } + + public HiveVarcharWritable getPrimitiveWritableObject(Object o) { + // check input object's length, if it doesn't match + // then output new writable with correct params. + if (o == null) { + return null; + } + HiveVarcharWritable writable = ((HiveVarcharWritable)o); + if (doesWritableMatchTypeParams((HiveVarcharWritable)o)) { + return writable; + } + + return getWritableWithParams(writable); + } + + private HiveVarchar getPrimitiveWithParams(HiveVarcharWritable val) { + HiveVarchar hv = new HiveVarchar(); + hv.setValue(val.getHiveVarchar(), getMaxLength()); + return hv; + } + + private HiveVarcharWritable getWritableWithParams(HiveVarcharWritable val) { + HiveVarcharWritable newValue = new HiveVarcharWritable(); + newValue.set(val, getMaxLength()); + return newValue; + } + + private boolean doesWritableMatchTypeParams(HiveVarcharWritable writable) { + return ParameterizedPrimitiveTypeUtils.doesWritableMatchTypeParams( + writable, (VarcharTypeParams) typeParams); + } + + private boolean doesPrimitiveMatchTypeParams(HiveVarchar value) { + return ParameterizedPrimitiveTypeUtils.doesPrimitiveMatchTypeParams( + value, (VarcharTypeParams) typeParams); + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + HiveVarcharWritable writable = (HiveVarcharWritable)o; + if (doesWritableMatchTypeParams((HiveVarcharWritable)o)) { + return new HiveVarcharWritable(writable); + } + return getWritableWithParams(writable); + } + + @Override + public Object set(Object o, HiveVarchar value) { + HiveVarcharWritable writable = (HiveVarcharWritable)o; + writable.set(value, getMaxLength()); + return o; + } + + @Override + public Object set(Object o, String value) { + HiveVarcharWritable writable = (HiveVarcharWritable)o; + writable.set(value, getMaxLength()); + return o; + } + + @Override + public Object create(HiveVarchar value) { + HiveVarcharWritable ret; + ret = new HiveVarcharWritable(); + ret.set(value, getMaxLength()); + return ret; + } + + public int getMaxLength() { + return typeParams != null ? ((VarcharTypeParams) typeParams).length : -1; + } +}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ParameterizedPrimitiveTypeUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ParameterizedPrimitiveTypeUtils.java?rev=1523463&r1=1523462&r2=1523463&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ParameterizedPrimitiveTypeUtils.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ParameterizedPrimitiveTypeUtils.java Sun Sep 15 17:23:53 2013 @@ -1,5 +1,9 @@ package org.apache.hadoop.hive.serde2.typeinfo; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; @@ -40,4 +44,28 @@ public class ParameterizedPrimitiveTypeU return oi.getTypeParams(); } + /** + * Utils for varchar type + */ + public static class HiveVarcharSerDeHelper { + public int maxLength; + public HiveVarcharWritable writable = new HiveVarcharWritable(); + + public HiveVarcharSerDeHelper(VarcharTypeParams typeParams) { + if (typeParams == null) { + throw new RuntimeException("varchar type used without type params"); + } + maxLength = typeParams.getLength(); + } + } + + public static boolean doesWritableMatchTypeParams(HiveVarcharWritable writable, + VarcharTypeParams typeParams) { + return (typeParams == null || typeParams.length >= writable.getCharacterLength()); + } + + public static boolean doesPrimitiveMatchTypeParams(HiveVarchar value, + VarcharTypeParams typeParams) { + return (typeParams == null || typeParams.length == value.getCharacterLength()); + } } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java?rev=1523463&r1=1523462&r2=1523463&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java Sun Sep 15 17:23:53 2013 @@ -49,6 +49,15 @@ public abstract class TypeInfo implement */ public abstract String getTypeName(); + /** + * String representing the qualified type name. + * Qualified types should override this method. + * @return + */ + public String getQualifiedName() { + return getTypeName(); + } + @Override public String toString() { return getTypeName(); Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java?rev=1523463&r1=1523462&r2=1523463&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java Sun Sep 15 17:23:53 2013 @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHa import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; @@ -67,6 +68,16 @@ public final class TypeInfoFactory { } } else { // No type params + + // Prevent creation of varchar TypeInfo with no length specification. + // This can happen if an old-style UDF uses a varchar type either as an + // argument or return type in an evaluate() function, or other instances + // of using reflection-based methods for retrieving a TypeInfo. + if (typeEntry.primitiveCategory == PrimitiveCategory.VARCHAR) { + LOG.error("varchar type used with no type params"); + throw new RuntimeException("varchar type used with no type params"); + } + result = new PrimitiveTypeInfo(parts.typeName); } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java?rev=1523463&r1=1523462&r2=1523463&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java Sun Sep 15 17:23:53 2013 @@ -29,6 +29,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; @@ -736,4 +738,24 @@ public final class TypeInfoUtils { } return true; } + + /** + * Return the character length of the type + * @param typeInfo + * @return + */ + public static int getCharacterLengthForType(PrimitiveTypeInfo typeInfo) { + switch (typeInfo.getPrimitiveCategory()) { + case STRING: + return HiveVarchar.MAX_VARCHAR_LENGTH; + case VARCHAR: + VarcharTypeParams varcharParams = (VarcharTypeParams) typeInfo.getTypeParams(); + if (varcharParams == null) { + throw new RuntimeException("varchar type used without type params"); + } + return varcharParams.getLength(); + default: + return 0; + } + } } Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeParams.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeParams.java?rev=1523463&view=auto ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeParams.java (added) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeParams.java Sun Sep 15 17:23:53 2013 @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.Serializable; + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.io.WritableUtils; + +public class VarcharTypeParams extends BaseTypeParams implements Serializable { + private static final long serialVersionUID = 1L; + + public int length; + + @Override + public void validateParams() throws SerDeException { + if (length < 1) { + throw new SerDeException("VARCHAR length must be positive"); + } + if (length > HiveVarchar.MAX_VARCHAR_LENGTH) { + throw new SerDeException("Length " + length + + " exceeds max varchar length of " + HiveVarchar.MAX_VARCHAR_LENGTH); + } + } + + @Override + public void populateParams(String[] params) throws SerDeException { + if (params.length != 1) { + throw new SerDeException("Invalid number of parameters for VARCHAR"); + } + try { + length = Integer.valueOf(params[0]); + } catch (NumberFormatException err) { + throw new SerDeException("Error setting VARCHAR length: " + err); + } + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("("); + sb.append(length); + sb.append(")"); + return sb.toString(); + } + + @Override + public void readFields(DataInput in) throws IOException { + length = WritableUtils.readVInt(in); + try { + validateParams(); + } catch (SerDeException err) { + throw new IOException(err); + } + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, length); + } + + public int getLength() { + return length; + } + + public void setLength(int len) { + length = len; + } + + @Override + public boolean hasCharacterMaximumLength() { + return true; + } + @Override + public int getCharacterMaximumLength() { + return length; + } +} Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestTypeInfoUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestTypeInfoUtils.java?rev=1523463&view=auto ============================================================================== --- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestTypeInfoUtils.java (added) +++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestTypeInfoUtils.java Sun Sep 15 17:23:53 2013 @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; + +public class TestTypeInfoUtils extends TestCase { + + static void parseTypeString(String typeString, boolean exceptionExpected) { + boolean caughtException = false; + try { + TypeInfoUtils.getTypeInfoFromTypeString(typeString); + } catch (IllegalArgumentException err) { + caughtException = true; + } + assertEquals("parsing typestring " + typeString, exceptionExpected, caughtException); + } + + public void testTypeInfoParser() { + String[] validTypeStrings = { + "int", + "string", + "varchar(10)", + "array<int>" + }; + + String[] invalidTypeStrings = { + "array<", + "varchar(123", + "varchar(123,", + "varchar()", + "varchar(" + }; + + for (String typeString : validTypeStrings) { + parseTypeString(typeString, false); + } + for (String typeString : invalidTypeStrings) { + parseTypeString(typeString, true); + } + } + + public void testVarcharNoParams() { + boolean caughtException = false; + try { + TypeInfoUtils.getTypeInfoFromTypeString("varchar"); + } catch (Exception err) { + caughtException = true; + } + assertEquals("varchar TypeInfo with no params should fail", true, caughtException); + } +}
