Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java Mon Oct 6 04:00:39 2014 @@ -19,10 +19,15 @@ package org.apache.hadoop.hive.serde2.ob import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharUtils; +import org.apache.hadoop.io.Text; +import org.apache.hive.common.util.HiveStringUtils; public class WritableHiveVarcharObjectInspector extends AbstractPrimitiveWritableObjectInspector implements SettableHiveVarcharObjectInspector { @@ -43,6 +48,12 @@ implements SettableHiveVarcharObjectInsp if (o == null) { return null; } + + if (o instanceof Text) { + String str = ((Text)o).toString(); + return new HiveVarchar(str, ((VarcharTypeInfo)typeInfo).getLength()); + } + HiveVarcharWritable writable = ((HiveVarcharWritable)o); if (doesWritableMatchTypeParams(writable)) { return writable.getHiveVarchar(); @@ -57,6 +68,14 @@ implements SettableHiveVarcharObjectInsp if (o == null) { return null; } + + if (o instanceof Text) { + String str = ((Text)o).toString(); + HiveVarcharWritable hcw = new HiveVarcharWritable(); + hcw.set(str, ((VarcharTypeInfo)typeInfo).getLength()); + return hcw; + } + HiveVarcharWritable writable = ((HiveVarcharWritable)o); if (doesWritableMatchTypeParams((HiveVarcharWritable)o)) { return writable;
Modified: hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java (original) +++ hive/branches/spark/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java Mon Oct 6 04:00:39 2014 @@ -22,6 +22,7 @@ import java.math.BigDecimal; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; public class HiveDecimalUtils { @@ -134,4 +135,25 @@ public class HiveDecimalUtils { } } + public static TypeInfo getDecimalTypeForPrimitiveCategories( + PrimitiveTypeInfo a, PrimitiveTypeInfo b) { + int prec1 = HiveDecimalUtils.getPrecisionForType(a); + int prec2 = HiveDecimalUtils.getPrecisionForType(b); + int scale1 = HiveDecimalUtils.getScaleForType(a); + int scale2 = HiveDecimalUtils.getScaleForType(b); + int intPart = Math.max(prec1 - scale1, prec2 - scale2); + int decPart = Math.max(scale1, scale2); + int prec = Math.min(intPart + decPart, HiveDecimal.MAX_PRECISION); + int scale = Math.min(decPart, HiveDecimal.MAX_PRECISION - intPart); + return TypeInfoFactory.getDecimalTypeInfo(prec, scale); + } + + public static DecimalTypeInfo getDecimalTypeForPrimitiveCategory(PrimitiveTypeInfo a) { + if (a instanceof DecimalTypeInfo) return (DecimalTypeInfo)a; + int prec = HiveDecimalUtils.getPrecisionForType(a); + int scale = HiveDecimalUtils.getScaleForType(a); + prec = Math.min(prec, HiveDecimal.MAX_PRECISION); + scale = Math.min(scale, HiveDecimal.MAX_PRECISION - (prec - scale)); + return TypeInfoFactory.getDecimalTypeInfo(prec, scale); + } } Modified: hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java (original) +++ hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java Mon Oct 6 04:00:39 2014 @@ -19,9 +19,11 @@ package org.apache.hadoop.hive.serde2.avro; import com.google.common.io.Resources; + import org.junit.Assert; import org.apache.avro.Schema; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; @@ -44,16 +46,28 @@ public class TestTypeInfoToSchema { private static Logger LOGGER = Logger.getLogger(TestTypeInfoToSchema.class); private static final List<String> COLUMN_NAMES = Arrays.asList("testCol"); - private static final TypeInfo STRING = TypeInfoFactory.getPrimitiveTypeInfo("string"); - private static final TypeInfo INT = TypeInfoFactory.getPrimitiveTypeInfo("int"); - private static final TypeInfo BOOLEAN = TypeInfoFactory.getPrimitiveTypeInfo("boolean"); - private static final TypeInfo LONG = TypeInfoFactory.getPrimitiveTypeInfo("bigint"); - private static final TypeInfo FLOAT = TypeInfoFactory.getPrimitiveTypeInfo("float"); - private static final TypeInfo DOUBLE = TypeInfoFactory.getPrimitiveTypeInfo("double"); - private static final TypeInfo BINARY = TypeInfoFactory.getPrimitiveTypeInfo("binary"); - private static final TypeInfo BYTE = TypeInfoFactory.getPrimitiveTypeInfo("tinyint"); - private static final TypeInfo SHORT = TypeInfoFactory.getPrimitiveTypeInfo("smallint"); - private static final TypeInfo VOID = TypeInfoFactory.getPrimitiveTypeInfo("void"); + private static final TypeInfo STRING = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.STRING_TYPE_NAME); + private static final TypeInfo INT = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.INT_TYPE_NAME); + private static final TypeInfo BOOLEAN = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.BOOLEAN_TYPE_NAME); + private static final TypeInfo LONG = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.BIGINT_TYPE_NAME); + private static final TypeInfo FLOAT = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.FLOAT_TYPE_NAME); + private static final TypeInfo DOUBLE = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.DOUBLE_TYPE_NAME); + private static final TypeInfo BINARY = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.BINARY_TYPE_NAME); + private static final TypeInfo BYTE = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.TINYINT_TYPE_NAME); + private static final TypeInfo SHORT = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.SMALLINT_TYPE_NAME); + private static final TypeInfo VOID = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.VOID_TYPE_NAME); + private static final TypeInfo DATE = TypeInfoFactory.getPrimitiveTypeInfo( + serdeConstants.DATE_TYPE_NAME); private static final int PRECISION = 4; private static final int SCALE = 2; private static final TypeInfo DECIMAL = TypeInfoFactory.getPrimitiveTypeInfo( @@ -205,6 +219,41 @@ public class TestTypeInfoToSchema { } @Test + public void createAvroCharSchema() { + final String specificSchema = "{" + + "\"type\":\"string\"," + + "\"logicalType\":\"char\"," + + "\"maxLength\":" + CHAR_LEN + "}"; + String expectedSchema = genSchema(specificSchema); + + Assert.assertEquals("Test for char's avro schema failed", + expectedSchema, getAvroSchemaString(CHAR)); + } + + @Test + public void createAvroVarcharSchema() { + final String specificSchema = "{" + + "\"type\":\"string\"," + + "\"logicalType\":\"varchar\"," + + "\"maxLength\":" + CHAR_LEN + "}"; + String expectedSchema = genSchema(specificSchema); + + Assert.assertEquals("Test for varchar's avro schema failed", + expectedSchema, getAvroSchemaString(VARCHAR)); + } + + @Test + public void createAvroDateSchema() { + final String specificSchema = "{" + + "\"type\":\"int\"," + + "\"logicalType\":\"date\"}"; + String expectedSchema = genSchema(specificSchema); + + Assert.assertEquals("Test for date in avro schema failed", + expectedSchema, getAvroSchemaString(DATE)); + } + + @Test public void createAvroListSchema() { ListTypeInfo listTypeInfo = new ListTypeInfo(); listTypeInfo.setListElementTypeInfo(STRING); @@ -313,6 +362,7 @@ public class TestTypeInfoToSchema { names.add("field11"); names.add("field12"); names.add("field13"); + names.add("field14"); structTypeInfo.setAllStructFieldNames(names); ArrayList<TypeInfo> typeInfos = new ArrayList<TypeInfo>(); typeInfos.add(STRING); @@ -327,6 +377,7 @@ public class TestTypeInfoToSchema { typeInfos.add(DOUBLE); typeInfos.add(BOOLEAN); typeInfos.add(DECIMAL); + typeInfos.add(DATE); typeInfos.add(VOID); structTypeInfo.setAllStructFieldTypeInfos(typeInfos); LOGGER.info("structTypeInfo is " + structTypeInfo); Modified: hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveCharWritable.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveCharWritable.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveCharWritable.java (original) +++ hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveCharWritable.java Mon Oct 6 04:00:39 2014 @@ -18,10 +18,20 @@ package org.apache.hadoop.hive.serde2.io; -import junit.framework.TestCase; +import com.google.code.tempusfugit.concurrency.annotations.*; +import com.google.code.tempusfugit.concurrency.*; +import org.junit.*; + +import static org.junit.Assert.*; import org.apache.hadoop.hive.common.type.HiveChar; -public class TestHiveCharWritable extends TestCase { +public class TestHiveCharWritable { + @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); + @Rule public RepeatingRule repeatingRule = new RepeatingRule(); + + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testConstructor() throws Exception { HiveCharWritable hcw1 = new HiveCharWritable(new HiveChar("abc", 5)); assertEquals("abc ", hcw1.toString()); @@ -30,6 +40,9 @@ public class TestHiveCharWritable extend assertEquals("abc ", hcw2.toString()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testSet() throws Exception { HiveCharWritable hcw1 = new HiveCharWritable(); @@ -70,18 +83,27 @@ public class TestHiveCharWritable extend assertEquals("ab", hcw1.getTextValue().toString()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testGetHiveChar() throws Exception { HiveCharWritable hcw = new HiveCharWritable(); hcw.set("abcd", 10); assertEquals("abcd ", hcw.getHiveChar().toString()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testGetCharacterLength() throws Exception { HiveCharWritable hcw = new HiveCharWritable(); hcw.set("abcd", 10); assertEquals(4, hcw.getCharacterLength()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testEnforceMaxLength() { HiveCharWritable hcw1 = new HiveCharWritable(); hcw1.set("abcdefghij", 10); @@ -92,6 +114,9 @@ public class TestHiveCharWritable extend assertEquals("abcde", hcw1.toString()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testComparison() throws Exception { HiveCharWritable hcw1 = new HiveCharWritable(); HiveCharWritable hcw2 = new HiveCharWritable(); Modified: hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java (original) +++ hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java Mon Oct 6 04:00:39 2014 @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.serde2.io; -import junit.framework.Assert; +import com.google.code.tempusfugit.concurrency.annotations.*; +import com.google.code.tempusfugit.concurrency.*; +import org.junit.*; +import static org.junit.Assert.*; import java.math.BigDecimal; import java.math.BigInteger; @@ -29,8 +32,6 @@ import java.util.ArrayList; import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hive.common.util.Decimal128FastBuffer; -import org.junit.Before; -import org.junit.Test; /** * Unit tests for tsting the fast allocation-free conversion @@ -38,14 +39,15 @@ import org.junit.Test; */ public class TestHiveDecimalWritable { - private Decimal128FastBuffer scratch; + @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); + @Rule public RepeatingRule repeatingRule = new RepeatingRule(); @Before public void setUp() throws Exception { - scratch = new Decimal128FastBuffer(); } private void doTestFastStreamForHiveDecimal(String valueString) { + Decimal128FastBuffer scratch = new Decimal128FastBuffer(); BigDecimal value = new BigDecimal(valueString); Decimal128 dec = new Decimal128(); dec.update(value); @@ -61,21 +63,23 @@ public class TestHiveDecimalWritable { BigDecimal readValue = hd.bigDecimalValue(); - Assert.assertEquals(value, readValue); + assertEquals(value, readValue); // Now test fastUpdate from the same serialized HiveDecimal Decimal128 decRead = new Decimal128().fastUpdateFromInternalStorage( witness.getInternalStorage(), (short) witness.getScale()); - Assert.assertEquals(dec, decRead); + assertEquals(dec, decRead); // Test fastUpdate from it's own (not fully compacted) serialized output Decimal128 decReadSelf = new Decimal128().fastUpdateFromInternalStorage( hdw.getInternalStorage(), (short) hdw.getScale()); - Assert.assertEquals(dec, decReadSelf); + assertEquals(dec, decReadSelf); } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testFastStreamForHiveDecimal() { doTestFastStreamForHiveDecimal("0"); @@ -217,7 +221,10 @@ public class TestHiveDecimalWritable { } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testHive6594() { + Decimal128FastBuffer scratch = new Decimal128FastBuffer(); String[] vs = new String[] { "-4033.445769230769", "6984454.211097692"}; @@ -236,7 +243,7 @@ public class TestHiveDecimalWritable { BigDecimal readValue = hd.bigDecimalValue(); - Assert.assertEquals(d.toBigDecimal().stripTrailingZeros(), + assertEquals(d.toBigDecimal().stripTrailingZeros(), readValue.stripTrailingZeros()); } } Modified: hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveVarcharWritable.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveVarcharWritable.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveVarcharWritable.java (original) +++ hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveVarcharWritable.java Mon Oct 6 04:00:39 2014 @@ -17,11 +17,21 @@ */ package org.apache.hadoop.hive.serde2.io; -import junit.framework.TestCase; +import com.google.code.tempusfugit.concurrency.annotations.*; +import com.google.code.tempusfugit.concurrency.*; +import org.junit.*; +import static org.junit.Assert.*; + import org.apache.hadoop.hive.common.type.HiveVarchar; import java.io.*; -public class TestHiveVarcharWritable extends TestCase { +public class TestHiveVarcharWritable { + @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); + @Rule public RepeatingRule repeatingRule = new RepeatingRule(); + + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testStringLength() throws Exception { HiveVarcharWritable vc1 = new HiveVarcharWritable(new HiveVarchar("0123456789", 10)); assertEquals(10, vc1.getCharacterLength()); @@ -54,6 +64,9 @@ public class TestHiveVarcharWritable ext assertEquals(6, vc1.getCharacterLength()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testEnforceLength() throws Exception { HiveVarcharWritable vc1 = new HiveVarcharWritable(new HiveVarchar("0123456789", 10)); assertEquals(10, vc1.getCharacterLength()); @@ -66,8 +79,11 @@ public class TestHiveVarcharWritable ext vc1.enforceMaxLength(8); assertEquals(8, vc1.getCharacterLength()); -} + } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testComparison() throws Exception { HiveVarcharWritable hc1 = new HiveVarcharWritable(new HiveVarchar("abcd", 20)); HiveVarcharWritable hc2 = new HiveVarcharWritable(new HiveVarchar("abcd", 20)); @@ -101,6 +117,9 @@ public class TestHiveVarcharWritable ext assertFalse(0 == hc2.compareTo(hc1)); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testStringValue() throws Exception { HiveVarcharWritable vc1 = new HiveVarcharWritable(new HiveVarchar("abcde", 20)); assertEquals("abcde", vc1.toString()); Modified: hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java (original) +++ hive/branches/spark/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java Mon Oct 6 04:00:39 2014 @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hive.serde2.io; +import com.google.code.tempusfugit.concurrency.annotations.*; +import com.google.code.tempusfugit.concurrency.*; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -32,15 +35,25 @@ import java.util.List; import java.util.Random; import java.util.TimeZone; -import junit.framework.TestCase; +import org.junit.*; +import static org.junit.Assert.*; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; -public class TestTimestampWritable extends TestCase { +public class TestTimestampWritable { + + @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); + @Rule public RepeatingRule repeatingRule = new RepeatingRule(); - private static DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static ThreadLocal<DateFormat> DATE_FORMAT = + new ThreadLocal<DateFormat>() { + @Override + protected synchronized DateFormat initialValue() { + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + } + }; private static final int HAS_DECIMAL_MASK = 0x80000000; @@ -64,14 +77,14 @@ public class TestTimestampWritable exten private static long parseToMillis(String s) { try { - return DATE_FORMAT.parse(s).getTime(); + return DATE_FORMAT.get().parse(s).getTime(); } catch (ParseException ex) { throw new RuntimeException(ex); } } - @Override - protected void setUp() { + @Before + public void setUp() { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); } @@ -252,6 +265,9 @@ public class TestTimestampWritable exten return tsw; } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testReverseNanos() { assertEquals(0, reverseNanos(0)); assertEquals(120000000, reverseNanos(21)); @@ -265,6 +281,8 @@ public class TestTimestampWritable exten * Test serializing and deserializing timestamps that can be represented by a number of seconds * from 0 to 2147483647 since the UNIX epoch. */ + @Test + @Concurrent(count=4) public void testTimestampsWithinPositiveIntRange() throws IOException { Random rand = new Random(294722773L); for (int i = 0; i < 10000; ++i) { @@ -281,6 +299,8 @@ public class TestTimestampWritable exten * Test timestamps that don't necessarily fit between 1970 and 2038. This depends on HIVE-4525 * being fixed. */ + @Test + @Concurrent(count=4) public void testTimestampsOutsidePositiveIntRange() throws IOException { Random rand = new Random(789149717L); for (int i = 0; i < 10000; ++i) { @@ -289,6 +309,8 @@ public class TestTimestampWritable exten } } + @Test + @Concurrent(count=4) public void testTimestampsInFullRange() throws IOException { Random rand = new Random(2904974913L); for (int i = 0; i < 10000; ++i) { @@ -296,6 +318,8 @@ public class TestTimestampWritable exten } } + @Test + @Concurrent(count=4) public void testToFromDouble() { Random rand = new Random(294729777L); for (int nanosPrecision = 0; nanosPrecision <= 4; ++nanosPrecision) { @@ -326,6 +350,8 @@ public class TestTimestampWritable exten return HiveDecimal.create(d); } + @Test + @Concurrent(count=4) public void testDecimalToTimestampRandomly() { Random rand = new Random(294729777L); for (int i = 0; i < 10000; ++i) { @@ -336,6 +362,9 @@ public class TestTimestampWritable exten } } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testDecimalToTimestampCornerCases() { Timestamp ts = new Timestamp(parseToMillis("1969-03-04 05:44:33")); assertEquals(0, ts.getTime() % 1000); @@ -347,6 +376,9 @@ public class TestTimestampWritable exten } } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testSerializationFormatDirectly() throws IOException { assertEquals("1970-01-01 00:00:00", fromIntAndVInts(0).toString()); assertEquals("1970-01-01 00:00:01", fromIntAndVInts(1).toString()); @@ -374,6 +406,9 @@ public class TestTimestampWritable exten -3210 - 1, seconds >> 31).toString()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testMaxSize() { // This many bytes are necessary to store the reversed nanoseconds. assertEquals(5, WritableUtils.getVIntSize(999999999)); @@ -396,6 +431,9 @@ public class TestTimestampWritable exten // Therefore, the maximum total size of a serialized timestamp is 4 + 5 + 4 = 13. } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testMillisToSeconds() { assertEquals(0, TimestampWritable.millisToSeconds(0)); assertEquals(-1, TimestampWritable.millisToSeconds(-1)); @@ -427,6 +465,9 @@ public class TestTimestampWritable exten return result < 0 ? -1 : (result > 0 ? 1 : 0); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testBinarySortable() { Random rand = new Random(5972977L); List<TimestampWritable> tswList = new ArrayList<TimestampWritable>(); Modified: hive/branches/spark/serde/src/test/resources/avro-struct.avsc URL: http://svn.apache.org/viewvc/hive/branches/spark/serde/src/test/resources/avro-struct.avsc?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/serde/src/test/resources/avro-struct.avsc (original) +++ hive/branches/spark/serde/src/test/resources/avro-struct.avsc Mon Oct 6 04:00:39 2014 @@ -4,11 +4,11 @@ "namespace":"", "doc":"struct<field1:string,field2:char(5),field3:varchar(5),field4:binary,field5:tinyint, field6:smallint,field7:int,field8:bigint,field9:float,field10:double,field11:boolean, -field12:decimal(4,2),field13:void>", +field12:decimal(4,2),field13:date,field14:void>", "fields":[ {"name":"field1","type":["null","string"],"doc":"string","default":null}, -{"name":"field2","type":["null","string"],"doc":"char(5)","default":null}, -{"name":"field3","type":["null","string"],"doc":"varchar(5)","default":null}, +{"name":"field2","type":["null",{"type":"string","logicalType":"char","maxLength":5}],"doc":"char(5)","default":null}, +{"name":"field3","type":["null",{"type":"string","logicalType":"varchar","maxLength":5}],"doc":"varchar(5)","default":null}, {"name":"field4","type":["null","bytes"],"doc":"binary","default":null}, {"name":"field5","type":["null","int"],"doc":"tinyint","default":null}, {"name":"field6","type":["null","int"],"doc":"smallint","default":null}, @@ -17,8 +17,8 @@ field12:decimal(4,2),field13:void>", {"name":"field9","type":["null","float"],"doc":"float","default":null}, {"name":"field10","type":["null","double"],"doc":"double","default":null}, {"name":"field11","type":["null","boolean"],"doc":"boolean","default":null}, -{"name":"field12","type":["null",{"type":"bytes","logicalType":"decimal","precision":4, -"scale":2}],"doc":"decimal(4,2)","default":null}, -{"name":"field13","type":"null","doc":"void","default":null} +{"name":"field12","type":["null",{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}],"doc":"decimal(4,2)","default":null}, +{"name":"field13","type":["null",{"type":"int","logicalType":"date"}],"doc":"date","default":null}, +{"name":"field14","type":"null","doc":"void","default":null} ] -} \ No newline at end of file +} Modified: hive/branches/spark/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java Mon Oct 6 04:00:39 2014 @@ -23,6 +23,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.HashMap; import java.util.Map; + import javax.security.auth.login.LoginException; import javax.security.sasl.Sasl; @@ -31,6 +32,7 @@ import org.apache.hadoop.hive.conf.HiveC import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.thrift.ThriftCLIService; import org.apache.thrift.TProcessorFactory; @@ -136,15 +138,17 @@ public class HiveAuthFactory { return transportFactory; } + /** + * Returns the thrift processor factory for HiveServer2 running in binary mode + * @param service + * @return + * @throws LoginException + */ public TProcessorFactory getAuthProcFactory(ThriftCLIService service) throws LoginException { - if ("http".equalsIgnoreCase(transportMode)) { - return HttpAuthUtils.getAuthProcFactory(service); + if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) { + return KerberosSaslHelper.getKerberosProcessorFactory(saslServer, service); } else { - if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) { - return KerberosSaslHelper.getKerberosProcessorFactory(saslServer, service); - } else { - return PlainSaslHelper.getPlainProcessorFactory(service); - } + return PlainSaslHelper.getPlainProcessorFactory(service); } } @@ -287,7 +291,9 @@ public class HiveAuthFactory { try { UserGroupInformation sessionUgi; if (ShimLoader.getHadoopShims().isSecurityEnabled()) { - sessionUgi = ShimLoader.getHadoopShims().createProxyUser(realUser); + KerberosName kerbName = new KerberosName(realUser); + String shortPrincipalName = kerbName.getServiceName(); + sessionUgi = ShimLoader.getHadoopShims().createProxyUser(shortPrincipalName); } else { sessionUgi = ShimLoader.getHadoopShims().createRemoteUser(realUser, null); } @@ -300,5 +306,5 @@ public class HiveAuthFactory { "Failed to validate proxy privilege of " + realUser + " for " + proxyUser, e); } } - + } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java Mon Oct 6 04:00:39 2014 @@ -22,17 +22,10 @@ import java.io.IOException; import java.security.PrivilegedExceptionAction; import org.apache.commons.codec.binary.Base64; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hive.service.cli.thrift.TCLIService; -import org.apache.hive.service.cli.thrift.TCLIService.Iface; -import org.apache.hive.service.cli.thrift.ThriftCLIService; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.HttpContext; -import org.apache.thrift.TProcessor; -import org.apache.thrift.TProcessorFactory; -import org.apache.thrift.transport.TTransport; import org.ietf.jgss.GSSContext; import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSManager; @@ -48,11 +41,7 @@ public final class HttpAuthUtils { public static final String AUTHORIZATION = "Authorization"; public static final String BASIC = "Basic"; public static final String NEGOTIATE = "Negotiate"; - - public static TProcessorFactory getAuthProcFactory(ThriftCLIService service) { - return new HttpCLIServiceProcessorFactory(service); - } - + /** * @return Stringified Base64 encoded kerberosAuthHeader on success */ @@ -62,7 +51,7 @@ public final class HttpAuthUtils { String serverPrincipal = getServerPrincipal(principal, host); // Uses the Ticket Granting Ticket in the UserGroupInformation return clientUGI.doAs( - new HttpKerberosClientAction(serverPrincipal, clientUGI.getShortUserName(), serverHttpUrl)); + new HttpKerberosClientAction(serverPrincipal, clientUGI.getUserName(), serverHttpUrl)); } /** @@ -87,26 +76,6 @@ public final class HttpAuthUtils { throw new UnsupportedOperationException("Can't initialize class"); } - public static class HttpCLIServiceProcessorFactory extends TProcessorFactory { - - private final ThriftCLIService service; - private final HiveConf hiveConf; - private final boolean isDoAsEnabled; - - public HttpCLIServiceProcessorFactory(ThriftCLIService service) { - super(null); - this.service = service; - hiveConf = service.getHiveConf(); - isDoAsEnabled = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS); - } - - @Override - public TProcessor getProcessor(TTransport trans) { - TProcessor baseProcessor = new TCLIService.Processor<Iface>(service); - return isDoAsEnabled ? new HttpCLIServiceUGIProcessor(baseProcessor) : baseProcessor; - } - } - public static class HttpKerberosClientAction implements PrivilegedExceptionAction<String> { public static final String HTTP_RESPONSE = "HTTP_RESPONSE"; Modified: hive/branches/spark/service/src/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java Mon Oct 6 04:00:39 2014 @@ -18,7 +18,6 @@ package org.apache.hive.service.auth; -import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Processor; import org.apache.hive.service.cli.thrift.TCLIService; import org.apache.hive.service.cli.thrift.TCLIService.Iface; import org.apache.thrift.TException; @@ -43,7 +42,7 @@ import org.slf4j.LoggerFactory; */ public class TSetIpAddressProcessor<I extends Iface> extends TCLIService.Processor<Iface> { - private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName()); + private static final Logger LOGGER = LoggerFactory.getLogger(TSetIpAddressProcessor.class.getName()); public TSetIpAddressProcessor(Iface iface) { super(iface); @@ -75,7 +74,7 @@ public class TSetIpAddressProcessor<I ex if (tSocket == null) { LOGGER.warn("Unknown Transport, cannot determine ipAddress"); } else { - THREAD_LOCAL_IP_ADDRESS.set(tSocket.getSocket().getInetAddress().toString()); + THREAD_LOCAL_IP_ADDRESS.set(tSocket.getSocket().getInetAddress().getHostAddress()); } } Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Mon Oct 6 04:00:39 2014 @@ -166,15 +166,20 @@ public class HiveSessionImpl implements IHiveFileProcessor processor = new GlobalHivercFileProcessor(); try { - if (hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION) != null) { - String hiverc = hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION) - + File.separator + SessionManager.HIVERCFILE; - if (new File(hiverc).exists()) { - LOG.info("Running global init file: " + hiverc); - int rc = processor.processFile(hiverc); + String hiverc = hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION); + if (hiverc != null) { + File hivercFile = new File(hiverc); + if (hivercFile.isDirectory()) { + hivercFile = new File(hivercFile, SessionManager.HIVERCFILE); + } + if (hivercFile.isFile()) { + LOG.info("Running global init file: " + hivercFile); + int rc = processor.processFile(hivercFile.getAbsolutePath()); if (rc != 0) { - LOG.warn("Failed on initializing global .hiverc file"); + LOG.error("Failed on initializing global .hiverc file"); } + } else { + LOG.debug("Global init file " + hivercFile + " does not exist"); } } } catch (IOException e) { Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/session/SessionManager.java Mon Oct 6 04:00:39 2014 @@ -229,6 +229,23 @@ public class SessionManager extends Comp return openSession(protocol, username, password, ipAddress, sessionConf, false, null); } + /** + * Opens a new session and creates a session handle. + * The username passed to this method is the effective username. + * If withImpersonation is true (==doAs true) we wrap all the calls in HiveSession + * within a UGI.doAs, where UGI corresponds to the effective user. + * @see org.apache.hive.service.cli.thrift.ThriftCLIService#getUserName() + * + * @param protocol + * @param username + * @param password + * @param ipAddress + * @param sessionConf + * @param withImpersonation + * @param delegationToken + * @return + * @throws HiveSQLException + */ public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress, Map<String, String> sessionConf, boolean withImpersonation, String delegationToken) throws HiveSQLException { Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java Mon Oct 6 04:00:39 2014 @@ -262,6 +262,16 @@ public abstract class ThriftCLIService e return clientIpAddress; } + /** + * Returns the effective username. + * 1. If hive.server2.allow.user.substitution = false: the username of the connecting user + * 2. If hive.server2.allow.user.substitution = true: the username of the end user, + * that the connecting user is trying to proxy for. + * This includes a check whether the connecting user is allowed to proxy for the end user. + * @param req + * @return + * @throws HiveSQLException + */ private String getUserName(TOpenSessionReq req) throws HiveSQLException { String userName = null; // Kerberos Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java Mon Oct 6 04:00:39 2014 @@ -31,6 +31,7 @@ import org.apache.hadoop.util.Shell; import org.apache.hive.service.auth.HiveAuthFactory; import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; import org.apache.hive.service.cli.CLIService; +import org.apache.hive.service.cli.thrift.TCLIService.Iface; import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup; import org.apache.thrift.TProcessor; import org.apache.thrift.TProcessorFactory; @@ -102,8 +103,7 @@ public class ThriftHttpCLIService extend // Thrift configs hiveAuthFactory = new HiveAuthFactory(hiveConf); - TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this); - TProcessor processor = processorFactory.getProcessor(null); + TProcessor processor = new TCLIService.Processor<Iface>(this); TProtocolFactory protocolFactory = new TBinaryProtocol.Factory(); // Set during the init phase of HiveServer2 if auth mode is kerberos // UGI for the hive/_HOST (kerberos) principal Modified: hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java (original) +++ hive/branches/spark/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java Mon Oct 6 04:00:39 2014 @@ -32,6 +32,7 @@ import org.apache.commons.codec.binary.S import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hive.service.auth.AuthenticationProviderFactory; import org.apache.hive.service.auth.AuthenticationProviderFactory.AuthMethods; import org.apache.hive.service.auth.HiveAuthFactory; @@ -219,7 +220,7 @@ public class ThriftHttpServlet extends T "provided by the client."); } else { - return getPrincipalWithoutRealm(gssContext.getSrcName().toString()); + return getPrincipalWithoutRealmAndHost(gssContext.getSrcName().toString()); } } catch (GSSException e) { @@ -237,8 +238,19 @@ public class ThriftHttpServlet extends T } private String getPrincipalWithoutRealm(String fullPrincipal) { - String names[] = fullPrincipal.split("[@]"); - return names[0]; + KerberosName fullKerberosName = new KerberosName(fullPrincipal); + String serviceName = fullKerberosName.getServiceName(); + String hostName = fullKerberosName.getHostName(); + String principalWithoutRealm = serviceName; + if (hostName != null) { + principalWithoutRealm = serviceName + "/" + hostName; + } + return principalWithoutRealm; + } + + private String getPrincipalWithoutRealmAndHost(String fullPrincipal) { + KerberosName fullKerberosName = new KerberosName(fullPrincipal); + return fullKerberosName.getServiceName(); } } Modified: hive/branches/spark/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java (original) +++ hive/branches/spark/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java Mon Oct 6 04:00:39 2014 @@ -150,7 +150,7 @@ public abstract class CLIServiceTest { client.closeOperation(opHandle); // Blocking execute - queryString = "SELECT ID FROM TEST_EXEC"; + queryString = "SELECT ID+1 FROM TEST_EXEC"; opHandle = client.executeStatement(sessionHandle, queryString, confOverlay); // Expect query to be completed now assertEquals("Query should be finished", @@ -225,27 +225,27 @@ public abstract class CLIServiceTest { /** * Execute an async query with default config */ - queryString = "SELECT ID FROM " + tableName; + queryString = "SELECT ID+1 FROM " + tableName; runQueryAsync(sessionHandle, queryString, confOverlay, OperationState.FINISHED, longPollingTimeout); /** * Execute an async query with long polling timeout set to 0 */ longPollingTimeout = 0; - queryString = "SELECT ID FROM " + tableName; + queryString = "SELECT ID+1 FROM " + tableName; runQueryAsync(sessionHandle, queryString, confOverlay, OperationState.FINISHED, longPollingTimeout); /** * Execute an async query with long polling timeout set to 500 millis */ longPollingTimeout = 500; - queryString = "SELECT ID FROM " + tableName; + queryString = "SELECT ID+1 FROM " + tableName; runQueryAsync(sessionHandle, queryString, confOverlay, OperationState.FINISHED, longPollingTimeout); /** * Cancellation test */ - queryString = "SELECT ID FROM " + tableName; + queryString = "SELECT ID+1 FROM " + tableName; opHandle = client.executeStatementAsync(sessionHandle, queryString, confOverlay); System.out.println("Cancelling " + opHandle); client.cancelOperation(opHandle); Modified: hive/branches/spark/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java (original) +++ hive/branches/spark/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java Mon Oct 6 04:00:39 2014 @@ -44,6 +44,7 @@ public class TestSessionGlobalInitFile e private ThriftCLIServiceClient client; private File initFile; private String tmpDir; + private HiveConf hiveConf; /** * This class is almost the same as EmbeddedThriftBinaryCLIService, @@ -86,7 +87,7 @@ public class TestSessionGlobalInitFile e FileUtils.writeLines(initFile, Arrays.asList(fileContent)); // set up service and client - HiveConf hiveConf = new HiveConf(); + hiveConf = new HiveConf(); hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION, initFile.getParentFile().getAbsolutePath()); service = new FakeEmbeddedThriftBinaryCLIService(hiveConf); @@ -102,11 +103,26 @@ public class TestSessionGlobalInitFile e @Test public void testSessionGlobalInitFile() throws Exception { - /** - * create session, and fetch the property set in global init file. Test if - * the global init file .hiverc is loaded correctly by checking the expected - * setting property. - */ + File tmpInitFile = new File(initFile.getParent(), "hiverc"); + Assert.assertTrue("Failed to rename " + initFile + " to " + tmpInitFile, + initFile.renameTo(tmpInitFile)); + initFile = tmpInitFile; + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION, + initFile.getAbsolutePath()); + doTestSessionGlobalInitFile(); + } + + @Test + public void testSessionGlobalInitDir() throws Exception { + doTestSessionGlobalInitFile(); + } + + /** + * create session, and fetch the property set in global init file. Test if + * the global init file .hiverc is loaded correctly by checking the expected + * setting property. + */ + private void doTestSessionGlobalInitFile() throws Exception { SessionHandle sessionHandle = client.openSession(null, null, null); verifyInitProperty("a", "1", sessionHandle); Modified: hive/branches/spark/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java URL: http://svn.apache.org/viewvc/hive/branches/spark/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java (original) +++ hive/branches/spark/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java Mon Oct 6 04:00:39 2014 @@ -177,7 +177,7 @@ public abstract class ThriftCLIServiceTe client.executeStatement(sessHandle, queryString, opConf); // Execute another query - queryString = "SELECT ID FROM TEST_EXEC_THRIFT"; + queryString = "SELECT ID+1 FROM TEST_EXEC_THRIFT"; OperationHandle opHandle = client.executeStatement(sessHandle, queryString, opConf); assertNotNull(opHandle); @@ -227,7 +227,7 @@ public abstract class ThriftCLIServiceTe client.executeStatement(sessHandle, queryString, opConf); // Execute another query - queryString = "SELECT ID FROM TEST_EXEC_ASYNC_THRIFT"; + queryString = "SELECT ID+1 FROM TEST_EXEC_ASYNC_THRIFT"; System.out.println("Will attempt to execute: " + queryString); opHandle = client.executeStatementAsync(sessHandle, queryString, opConf); Modified: hive/branches/spark/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java URL: http://svn.apache.org/viewvc/hive/branches/spark/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original) +++ hive/branches/spark/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Mon Oct 6 04:00:39 2014 @@ -918,4 +918,14 @@ public class Hadoop20Shims implements Ha public boolean hasStickyBit(FsPermission permission) { return false; // not supported } + + @Override + public boolean supportTrashFeature() { + return false; + } + + @Override + public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { + return null; + } } Modified: hive/branches/spark/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java URL: http://svn.apache.org/viewvc/hive/branches/spark/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original) +++ hive/branches/spark/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Mon Oct 6 04:00:39 2014 @@ -534,6 +534,16 @@ public class Hadoop20SShims extends Hado @Override public boolean hasStickyBit(FsPermission permission) { - return false; // not supported + return false; + } + + @Override + public boolean supportTrashFeature() { + return false; + } + + @Override + public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { + return null; } } Modified: hive/branches/spark/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java URL: http://svn.apache.org/viewvc/hive/branches/spark/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original) +++ hive/branches/spark/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Mon Oct 6 04:00:39 2014 @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.ProxyFileSystem; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; @@ -835,4 +836,15 @@ public class Hadoop23Shims extends Hadoo public boolean hasStickyBit(FsPermission permission) { return permission.getStickyBit(); } + + @Override + public boolean supportTrashFeature() { + return true; + } + + @Override + public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { + TrashPolicy tp = TrashPolicy.getInstance(conf, fs, fs.getHomeDirectory()); + return tp.getCurrentTrashDir(); + } } Modified: hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java URL: http://svn.apache.org/viewvc/hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1629563&r1=1629562&r2=1629563&view=diff ============================================================================== --- hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original) +++ hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Mon Oct 6 04:00:39 2014 @@ -721,4 +721,14 @@ public interface HadoopShims { * @return sticky bit */ boolean hasStickyBit(FsPermission permission); + + /** + * @return True if the current hadoop supports trash feature. + */ + boolean supportTrashFeature(); + + /** + * @return Path to HDFS trash, if current hadoop supports trash feature. Null otherwise. + */ + Path getCurrentTrashPath(Configuration conf, FileSystem fs); }
