Repository: hive Updated Branches: refs/heads/master 561dbe3b9 -> 5a0a15f3d
HIVE-15718: Fix the NullPointer problem caused by split phase(Colin Ma, reviewed by Ferdinand Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5a0a15f3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5a0a15f3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5a0a15f3 Branch: refs/heads/master Commit: 5a0a15f3de54788509665e123e5096846f977b9f Parents: 561dbe3 Author: Ferdinand Xu <[email protected]> Authored: Wed Feb 8 01:59:23 2017 +0800 Committer: Ferdinand Xu <[email protected]> Committed: Wed Feb 8 01:59:23 2017 +0800 ---------------------------------------------------------------------- .../vector/VectorizedParquetRecordReader.java | 8 ++++++-- .../io/parquet/TestVectorizedColumnReader.java | 21 ++++++++++++++++++++ .../parquet/VectorizedColumnReaderTestBase.java | 2 +- 3 files changed, 28 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/5a0a15f3/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index 699de59..b6a1a7a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -82,7 +82,7 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase /** * The number of rows that have been returned. */ - private long rowsReturned; + private long rowsReturned = 0; /** * The number of rows that have been reading, including the current in flight row group. @@ -93,7 +93,7 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase * The total number of rows this RecordReader will eventually read. The sum of the * rows of all the row groups. */ - protected long totalRowCount; + protected long totalRowCount = 0; @VisibleForTesting public VectorizedParquetRecordReader( @@ -129,6 +129,10 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase public void initialize( InputSplit oldSplit, JobConf configuration) throws IOException, InterruptedException { + // the oldSplit may be null during the split phase + if (oldSplit == null) { + return; + } jobConf = configuration; ParquetMetadata footer; List<BlockMetaData> blocks; http://git-wip-us.apache.org/repos/asf/hive/blob/5a0a15f3/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java index 33567eb..670bfa6 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java @@ -18,12 +18,20 @@ package org.apache.hadoop.hive.ql.io.parquet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.InputSplit; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; +import static junit.framework.TestCase.assertFalse; + public class TestVectorizedColumnReader extends VectorizedColumnReaderTestBase { static boolean isDictionaryEncoding = false; @@ -88,4 +96,17 @@ public class TestVectorizedColumnReader extends VectorizedColumnReaderTestBase { public void decimalRead() throws Exception { decimalRead(isDictionaryEncoding); } + + @Test + public void testNullSplitForParquetReader() throws Exception { + Configuration conf = new Configuration(); + conf.set(IOConstants.COLUMNS,"int32_field"); + conf.set(IOConstants.COLUMNS_TYPES,"int"); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); + HiveConf.setVar(conf, HiveConf.ConfVars.PLAN, "//tmp"); + initialVectorizedRowBatchCtx(conf); + VectorizedParquetRecordReader reader = + new VectorizedParquetRecordReader((InputSplit)null, new JobConf(conf)); + assertFalse(reader.next(reader.createKey(), reader.createValue())); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/5a0a15f3/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java index 833cfdb..f537cee 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java @@ -295,7 +295,7 @@ public class VectorizedColumnReaderTestBase { writer.close(); } - private void initialVectorizedRowBatchCtx(Configuration conf) throws HiveException { + protected void initialVectorizedRowBatchCtx(Configuration conf) throws HiveException { MapWork mapWork = new MapWork(); VectorizedRowBatchCtx rbCtx = new VectorizedRowBatchCtx(); rbCtx.init(createStructObjectInspector(conf), new String[0]);
