[CARBONDATA-2105] Fixed bug for null values when group by column is present as dictionary_include
Refactored code to resolve issue of null values when group by column is present as dictionary_include. This closes #1917 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/11f23714 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/11f23714 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/11f23714 Branch: refs/heads/branch-1.3 Commit: 11f23714cd7ff49759a57e89ae947fde56a40c06 Parents: 55bffbe Author: SangeetaGulia <sangeeta.gu...@knoldus.in> Authored: Fri Feb 2 22:41:24 2018 +0530 Committer: kumarvishal <kumarvishal.1...@gmail.com> Committed: Sat Feb 3 18:21:18 2018 +0530 ---------------------------------------------------------------------- .../preaggregate/TestPreAggCreateCommand.scala | 4 ++-- .../TestPreAggregateTableSelection.scala | 17 +++++++++++++++++ .../command/carbonTableSchemaCommon.scala | 3 ++- 3 files changed, 21 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/11f23714/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala index 38ab9ae..5d0f61b 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala @@ -200,8 +200,8 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll { sql("create datamap agg0 on table mainTable using 'preaggregate' as select column1, count(column1),column6, count(column6) from maintable group by column6,column1") val df = sql("select * from maintable_agg0") val carbontable = getCarbontable(df.queryExecution.analyzed) - assert(carbontable.getAllMeasures.size()==1) - assert(carbontable.getAllDimensions.size()==4) + assert(carbontable.getAllMeasures.size()==2) + assert(carbontable.getAllDimensions.size()==2) carbontable.getAllDimensions.asScala.foreach{ f => assert(f.getEncoder.contains(Encoding.DICTIONARY)) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/11f23714/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala index 5fb7b02..19d4abe 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala @@ -38,6 +38,7 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll { sql("drop table if exists agg5") sql("drop table if exists agg6") sql("drop table if exists agg7") + sql("DROP TABLE IF EXISTS maintabledict") sql("CREATE TABLE mainTable(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format'") sql("create datamap agg0 on table mainTable using 'preaggregate' as select name from mainTable group by name") sql("create datamap agg1 on table mainTable using 'preaggregate' as select name,sum(age) from mainTable group by name") @@ -320,11 +321,27 @@ test("test PreAggregate table selection with timeseries and normal together") { val df = sql("select avg(age) from mainTableavg") preAggTableValidator(df.queryExecution.analyzed, "mainTableavg_agg0") } + + test("test PreAggregate table selection for avg with maintable containing dictionary include for group by column") { + sql( + "create table maintabledict(year int,month int,name string,salary int,dob string) stored" + + " by 'carbondata' tblproperties('DICTIONARY_INCLUDE'='year')") + sql("insert into maintabledict select 10,11,'x',12,'2014-01-01 00:00:00'") + sql("insert into maintabledict select 10,11,'x',12,'2014-01-01 00:00:00'") + sql( + "create datamap aggdict on table maintabledict using 'preaggregate' as select year,avg(year) from " + + "maintabledict group by year") + val df = sql("select year,avg(year) from maintabledict group by year") + checkAnswer(df, Seq(Row(10,10.0))) + } + + override def afterAll: Unit = { sql("drop table if exists mainTable") sql("drop table if exists mainTable_avg") sql("drop table if exists lineitem") sql("DROP TABLE IF EXISTS maintabletime") + sql("DROP TABLE IF EXISTS maintabledict") } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/11f23714/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala index 9a0098e..bc84e04 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala @@ -544,7 +544,8 @@ class TableNewProcessor(cm: TableModel) { val encoders = if (getEncoderFromParent(field)) { isAggFunPresent = cm.dataMapRelation.get.get(field).get.aggregateFunction.equalsIgnoreCase("sum") || - cm.dataMapRelation.get.get(field).get.aggregateFunction.equals("avg") + cm.dataMapRelation.get.get(field).get.aggregateFunction.equals("avg") || + cm.dataMapRelation.get.get(field).get.aggregateFunction.equals("count") if(!isAggFunPresent) { cm.parentTable.get.getColumnByName( cm.parentTable.get.getTableName,
