Author: hashutosh
Date: Thu Dec 12 22:28:47 2013
New Revision: 1550554
URL: http://svn.apache.org/r1550554
Log:
HIVE-5991 : ORC RLEv2 fails with ArrayIndexOutOfBounds exception for
PATCHED_BLOB encoding (Prasanth J via Owen Omalley)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java?rev=1550554&r1=1550553&r2=1550554&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
Thu Dec 12 22:28:47 2013
@@ -562,7 +562,7 @@ class RunLengthIntegerWriterV2 implement
private void preparePatchedBlob() {
// mask will be max value beyond which patch will be generated
- int mask = (1 << brBits95p) - 1;
+ long mask = (1L << brBits95p) - 1;
// since we are considering only 95 percentile, the size of gap and
// patch array can contain only be 5% values
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java?rev=1550554&r1=1550553&r2=1550554&view=diff
==============================================================================
---
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
(original)
+++
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
Thu Dec 12 22:28:47 2013
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@@ -46,6 +47,14 @@ import com.google.common.primitives.Long
public class TestNewIntegerEncoding {
+ public static class TSRow {
+ Timestamp ts;
+
+ public TSRow(Timestamp ts) {
+ this.ts = ts;
+ }
+ }
+
public static class Row {
Integer int1;
Long long1;
@@ -878,6 +887,68 @@ public class TestNewIntegerEncoding {
}
@Test
+ public void testPatchedBaseTimestamp() throws Exception {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector =
ObjectInspectorFactory.getReflectionObjectInspector(TSRow.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+
+ Writer writer = OrcFile.createWriter(testFilePath,
+
OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000));
+
+ List<Timestamp> tslist = Lists.newArrayList();
+ tslist.add(Timestamp.valueOf("9999-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("1999-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("1995-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2010-03-02 00:00:00"));
+ tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("1996-08-02 00:00:00"));
+ tslist.add(Timestamp.valueOf("1998-11-02 00:00:00"));
+ tslist.add(Timestamp.valueOf("2008-10-02 00:00:00"));
+ tslist.add(Timestamp.valueOf("1993-08-02 00:00:00"));
+ tslist.add(Timestamp.valueOf("2008-01-02 00:00:00"));
+ tslist.add(Timestamp.valueOf("2007-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2008-10-02 00:00:00"));
+ tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2008-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("1994-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2001-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2000-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2000-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2011-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
+ tslist.add(Timestamp.valueOf("1974-01-01 00:00:00"));
+
+ for (Timestamp ts : tslist) {
+ writer.addRow(new TSRow(ts));
+ }
+
+ writer.close();
+
+ Reader reader = OrcFile.createReader(fs, testFilePath);
+ RecordReader rows = reader.rows(null);
+ int idx = 0;
+ while (rows.hasNext()) {
+ Object row = rows.next(null);
+ assertEquals(tslist.get(idx++).getNanos(),
+ ((Timestamp) ((OrcStruct) row).getFieldValue(0)).getNanos());
+ }
+ }
+
+ @Test
public void testSeek() throws Exception {
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
