Repository: carbondata Updated Branches: refs/heads/master deabeffd9 -> e2f19979b
[CARBONDATA-1774] [PrestoIntegration] Not able to fetch data from a table with Boolean data type in presto Not able to fetch data from a Carbon table with Boolean data type in presto This closes #1709 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e2f19979 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e2f19979 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e2f19979 Branch: refs/heads/master Commit: e2f19979b6be2a38de936da316ed91d9b9f6bfee Parents: deabeff Author: anubhav100 <anubhav.ta...@knoldus.in> Authored: Tue Dec 19 15:33:06 2017 +0530 Committer: chenliang613 <chenliang...@huawei.com> Committed: Thu Jan 4 16:50:42 2018 +0800 ---------------------------------------------------------------------- .../presto/CarbonVectorizedRecordReader.java | 17 ++- .../presto/readers/BooleanStreamReader.java | 107 +++++++++++++++++++ .../presto/readers/StreamReaders.java | 4 + .../presto/src/test/resources/alldatatype.csv | 22 ++-- .../integrationtest/PrestoAllDataTypeTest.scala | 15 +++ .../presto/util/CarbonDataStoreCreator.scala | 13 ++- 6 files changed, 157 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java index 910ccf4..9a8f8c5 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java @@ -186,18 +186,17 @@ class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> { } } - for (int i = 0; i < queryMeasures.size(); i++) { - QueryMeasure msr = queryMeasures.get(i); + for (QueryMeasure msr : queryMeasures) { DataType dataType = msr.getMeasure().getDataType(); - if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) { - fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), - msr.getMeasure().getDataType()); + if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT + || dataType == DataTypes.LONG) { + fields[msr.getQueryOrder()] = + new StructField(msr.getColumnName(), msr.getMeasure().getDataType()); } else if (DataTypes.isDecimal(dataType)) { - fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), - msr.getMeasure().getDataType()); + fields[msr.getQueryOrder()] = + new StructField(msr.getColumnName(), msr.getMeasure().getDataType()); } else { - fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), - DataTypes.DOUBLE); + fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java new file mode 100644 index 0000000..2a45e72 --- /dev/null +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.presto.readers; + +import java.io.IOException; + +import org.apache.carbondata.core.cache.dictionary.Dictionary; +import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.util.DataTypeUtil; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.BlockBuilderStatus; +import com.facebook.presto.spi.type.Type; + +public class BooleanStreamReader extends AbstractStreamReader { + + private boolean isDictionary; + private Dictionary dictionary; + + public BooleanStreamReader() { + + } + + public BooleanStreamReader(boolean isDictionary, Dictionary dictionary) { + this.isDictionary = isDictionary; + this.dictionary = dictionary; + } + + public Block readBlock(Type type) throws IOException { + int numberOfRows = 0; + BlockBuilder builder = null; + if (isVectorReader) { + numberOfRows = batchSize; + builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows); + if (columnVector != null) { + if(columnVector.anyNullsSet()) { + handleNullInVector(type, numberOfRows, builder); + } + else { + populateVector(type, numberOfRows, builder); + } + } + + } else { + numberOfRows = streamData.length; + builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows); + if (streamData != null) { + for (int i = 0; i < numberOfRows; i++) { + type.writeBoolean(builder, byteToBoolean(streamData[i])); + } + } + } + + return builder.build(); + } + + private void handleNullInVector(Type type, int numberOfRows, BlockBuilder builder) { + for (int i = 0; i < numberOfRows; i++) { + if (columnVector.isNullAt(i)) { + builder.appendNull(); + } else { + type.writeBoolean(builder, byteToBoolean(columnVector.getData(i))); + } + } + } + + private void populateVector(Type type, int numberOfRows, BlockBuilder builder) { + if(isDictionary) { + for (int i = 0; i < numberOfRows; i++) { + int value = (int) columnVector.getData(i); + Object data = DataTypeUtil + .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.BOOLEAN); + if (data != null) { + type.writeBoolean(builder,(boolean) data); + } else { + builder.appendNull(); + } + } + } + else { + for (int i = 0; i < numberOfRows; i++) { + type.writeBoolean(builder, byteToBoolean(columnVector.getData(i))); + } + } + } + private Boolean byteToBoolean(Object value){ + byte byteValue = (byte)value; + return byteValue == 1; + } +} + http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java index b9b8b7e..1ad3d28 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java @@ -61,6 +61,8 @@ public final class StreamReaders { } else { return new SliceStreamReader(true, dictionarySliceArrayBlock); } + }else if (javaType == boolean.class) { + return new BooleanStreamReader(true,dictionary); } else { return new ObjectStreamReader(); } @@ -85,6 +87,8 @@ public final class StreamReaders { } else { return new SliceStreamReader(); } + }else if (javaType == boolean.class) { + return new BooleanStreamReader(); } else { return new ObjectStreamReader(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/test/resources/alldatatype.csv ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/resources/alldatatype.csv b/integration/presto/src/test/resources/alldatatype.csv index ba6156e..7201542 100644 --- a/integration/presto/src/test/resources/alldatatype.csv +++ b/integration/presto/src/test/resources/alldatatype.csv @@ -1,11 +1,11 @@ -ID,date,country,name,phonetype,serialname,salary,bonus,monthlyBonus,dob,shortfield -1,2015-07-23,china,anubhav,phone197,ASD69643,5000000.00,1234.444,12.1234,2016-04-14 15/00/09,10 -2,2015-07-24,china,jatin,phone756,ASD42892,150010.999,1234.5555,15.13,2016-04-14 15:00:09,10 -3,2015-07-25,china,liang,phone1904,ASD37014,15002.110,600.777,16.181,2016-01-14 15:07:09,8 -4,2015-07-26,china,prince,phone2435,ASD66902,15003.00,9999.999,17.3654,1992-04-14 13:00:09,4 -5,2015-07-27,china,bhavya,phone2441,ASD90633,15004.00,5000.999,12.11,2010-06-19 14:10:06,11 -6,2015-07-28,china,akash,phone294,ASD59961,15005.00,500.59,18.65,2013-07-19 12:10:08,18 -7,2015-07-29,china,sahil,phone610,ASD14875,15006.00,500.99,,19.65,2007-04-19 11:10:06,17 -8,2015-07-30,china,geetika,phone1848,ASD57308,15007.500,500.88,200.97,2008-09-21 11:10:06,10 -9,2015-07-18,china,ravindra,phone706,ASD86717,15008.00,700.999,45.25,2009-06-19 15:10:06,1 -9,2015/07/18,china,jitesh,phone706,ASD86717,15008.00,500.414,11.655,2001-08-29 13:09:03,12 +ID,date,country,name,phonetype,serialname,salary,bonus,monthlyBonus,dob,shortfield,isCurrentEmployee +1,2015-07-23,china,anubhav,phone197,ASD69643,5000000.00,1234.444,12.1234,2016-04-14 15/00/09,10,true +2,2015-07-24,china,jatin,phone756,ASD42892,150010.999,1234.5555,15.13,2016-04-14 15:00:09,10,null +3,2015-07-25,china,liang,phone1904,ASD37014,15002.110,600.777,16.181,2016-01-14 15:07:09,8,true +4,2015-07-26,china,prince,phone2435,ASD66902,15003.00,9999.999,17.3654,1992-04-14 13:00:09,4,true +5,2015-07-27,china,bhavya,phone2441,ASD90633,15004.00,5000.999,12.11,2010-06-19 14:10:06,11,true +6,2015-07-28,china,akash,phone294,ASD59961,15005.00,500.59,18.65,2013-07-19 12:10:08,18,false +7,2015-07-29,china,sahil,phone610,ASD14875,15006.00,500.99,,19.65,2007-04-19 11:10:06,17,false +8,2015-07-30,china,geetika,phone1848,ASD57308,15007.500,500.88,200.97,2008-09-21 11:10:06,10,true +9,2015-07-18,china,ravindra,phone706,ASD86717,15008.00,700.999,45.25,2009-06-19 15:10:06,1,true +9,2015/07/18,china,jitesh,phone706,ASD86717,15008.00,500.414,11.655,2001-08-29 13:09:03,12,true http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala index 153ca5e..49da227 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala @@ -435,4 +435,19 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "2")) assert(actualResult.toString() equals expectedResult.toString()) } + test("test the boolean data type") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT isCurrentEmployee AS RESULT FROM TESTDB.TESTTABLE WHERE ID=1") + assert(actualResult.head("RESULT").toString.toBoolean) + } + test("test the boolean data type for null value") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT id AS RESULT FROM TESTDB.TESTTABLE WHERE isCurrentEmployee is null") + assert(actualResult.head("RESULT").toString.toInt==2) + } + test("test the boolean data type for not null value with filter ") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT id AS RESULT FROM TESTDB.TESTTABLE WHERE isCurrentEmployee is NOT null AND ID>8") + assert(actualResult.head("RESULT").toString.toInt==9) + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala index 5a0adf5..7b5c311 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala @@ -131,7 +131,7 @@ object CarbonDataStoreCreator { "true") loadModel.setMaxColumns("15") loadModel.setCsvHeader( - "ID,date,country,name,phonetype,serialname,salary,bonus,monthlyBonus,dob,shortField") + "ID,date,country,name,phonetype,serialname,salary,bonus,monthlyBonus,dob,shortField,isCurrentEmployee") loadModel.setCsvHeaderColumns(loadModel.getCsvHeader.split(",")) loadModel.setTaskNo("0") loadModel.setSegmentId("0") @@ -302,6 +302,17 @@ object CarbonDataStoreCreator { shortField.setColumnReferenceId(shortField.getColumnUniqueId) columnSchemas.add(shortField) + val isCurrentEmployee: ColumnSchema = new ColumnSchema() + isCurrentEmployee.setColumnName("isCurrentEmployee") + isCurrentEmployee.setColumnar(true) + isCurrentEmployee.setDataType(DataTypes.BOOLEAN) + isCurrentEmployee.setEncodingList(invertedIndexEncoding) + isCurrentEmployee.setColumnUniqueId(UUID.randomUUID().toString) + isCurrentEmployee.setDimensionColumn(false) + isCurrentEmployee.setColumnGroup(11) + isCurrentEmployee.setColumnReferenceId(isCurrentEmployee.getColumnUniqueId) + columnSchemas.add(isCurrentEmployee) + tableSchema.setListOfColumns(columnSchemas) val schemaEvol: SchemaEvolution = new SchemaEvolution() schemaEvol.setSchemaEvolutionEntryList(
