[CARBONDATA-3081] Fixed NPE for boolean type column with null value Problem: NPE thrown when boolean type column has null values.
Solution: check for null values before converting byte to boolean. This closes #2901 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b69f0fc8 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b69f0fc8 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b69f0fc8 Branch: refs/heads/branch-1.5 Commit: b69f0fc87da1cd50191d055658d47c3f1ea3559b Parents: 413cd80 Author: kunal642 <kunalkapoor...@gmail.com> Authored: Mon Nov 5 18:46:44 2018 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Wed Nov 21 22:43:15 2018 +0530 ---------------------------------------------------------------------- .../core/metadata/datatype/DecimalType.java | 2 +- .../util/CarbonVectorizedRecordReader.java | 19 +++++--- .../carbondata/sdk/file/CarbonReaderTest.java | 49 ++++++++++++++++++++ 3 files changed, 63 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/b69f0fc8/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java index b4bc20c..a7f7a4e 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java @@ -23,7 +23,7 @@ public class DecimalType extends DataType { private int scale; // create a decimal type object with specified precision and scale - public DecimalType(int precision, int scale) { + DecimalType(int precision, int scale) { super(DataTypes.DECIMAL_TYPE_ID, 8, "DECIMAL", -1); this.precision = precision; this.scale = scale; http://git-wip-us.apache.org/repos/asf/carbondata/blob/b69f0fc8/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java index 9d3d7d6..7720434 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java @@ -29,7 +29,6 @@ import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; -import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.metadata.datatype.StructField; import org.apache.carbondata.core.scan.executor.QueryExecutor; import org.apache.carbondata.core.scan.executor.QueryExecutorFactory; @@ -149,7 +148,8 @@ public class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> { new StructField(msr.getColumnName(), msr.getMeasure().getDataType()); } else if (DataTypes.isDecimal(dataType)) { fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), - new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale())); + DataTypes.createDecimalType(msr.getMeasure().getPrecision(), + msr.getMeasure().getScale())); } else { fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE); } @@ -171,13 +171,20 @@ public class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> { rowCount += 1; Object[] row = new Object[carbonColumnarBatch.columnVectors.length]; for (int i = 0; i < carbonColumnarBatch.columnVectors.length; i ++) { + Object data = carbonColumnarBatch.columnVectors[i].getData(batchIdx - 1); if (carbonColumnarBatch.columnVectors[i].getType() == DataTypes.STRING || carbonColumnarBatch.columnVectors[i].getType() == DataTypes.VARCHAR) { - byte[] data = (byte[]) carbonColumnarBatch.columnVectors[i].getData(batchIdx - 1); - row[i] = ByteUtil.toString(data, 0, data.length); + if (data == null) { + row[i] = null; + } else { + row[i] = ByteUtil.toString((byte[]) data, 0, (((byte[]) data).length)); + } } else if (carbonColumnarBatch.columnVectors[i].getType() == DataTypes.BOOLEAN) { - byte data = (byte) carbonColumnarBatch.columnVectors[i].getData(batchIdx - 1); - row[i] = ByteUtil.toBoolean(data); + if (data == null) { + row[i] = null; + } else { + row[i] = ByteUtil.toBoolean((byte) data); + } } else { row[i] = carbonColumnarBatch.columnVectors[i].getData(batchIdx - 1); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/b69f0fc8/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index 0767264..9add0b9 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -2023,4 +2023,53 @@ public class CarbonReaderTest extends TestCase { } } + @Test + public void testReadingNullValues() { + String path = "./testWriteFiles"; + try { + FileUtils.deleteDirectory(new File(path)); + + Field[] fields = new Field[2]; + fields[0] = new Field("stringField", DataTypes.STRING); + fields[1] = new Field("booleanField", DataTypes.BOOLEAN); + CarbonWriter writer = CarbonWriter.builder() + .outputPath(path) + .withCsvInput(new Schema(fields)) + .writtenBy("CarbonReaderTest") + .build(); + + for (int i = 0; i < 2; i++) { + String[] row2 = new String[]{ + "robot" + (i % 10), + "", + }; + writer.write(row2); + } + writer.close(); + + // Read data + CarbonReader reader = CarbonReader + .builder(path, "_temp") + .build(); + + int i = 0; + while (reader.hasNext()) { + reader.readNextRow(); + i++; + } + assert (i == 2); + reader.close(); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } finally { + try { + FileUtils.deleteDirectory(new File(path)); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + } + }