[CARBONDATA-2029] Fixed Pre Aggregate table issue with expression Query with expression is giving wrong result
This closes #1803 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/04d40ccb Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/04d40ccb Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/04d40ccb Branch: refs/heads/carbonstore Commit: 04d40ccb911199bad4825292214c5ba34910f12a Parents: 0d23461 Author: kumarvishal <[email protected]> Authored: Mon Jan 15 16:26:36 2018 +0530 Committer: kunal642 <[email protected]> Committed: Tue Jan 16 14:48:22 2018 +0530 ---------------------------------------------------------------------- .../TestPreAggregateTableSelection.scala | 5 +++++ .../spark/sql/hive/CarbonPreAggregateRules.scala | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/04d40ccb/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala index 9bbba3a..559e91f 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala @@ -262,6 +262,11 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll { preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0") } + test("Test query with math operation hitting fact table") { + val df = sql("select sum(id)+count(id) from maintable") + preAggTableValidator(df.queryExecution.analyzed, "maintable") + } + override def afterAll: Unit = { sql("drop table if exists mainTable") sql("drop table if exists lineitem") http://git-wip-us.apache.org/repos/asf/carbondata/blob/04d40ccb/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala index de554c5..299ed4d 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala @@ -1013,6 +1013,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule carbonTable: CarbonTable, queryColumns: scala.collection.mutable.HashSet[QueryColumn], aggreagteExps: scala.collection.mutable.HashSet[AggregateExpression]): Boolean = { + var isValid = true groupByExpression foreach { expression => extractColumnFromExpression(expression, queryColumns, carbonTable) } @@ -1025,13 +1026,13 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule carbonTable); case Alias(attr: AggregateExpression, _) => if (attr.isDistinct) { - return false + isValid = false } val aggExp = PreAggregateUtil.validateAggregateFunctionAndGetFields(attr) if (aggExp.nonEmpty) { aggreagteExps ++= aggExp } else { - return false + isValid = false } case Alias(expression: Expression, _) => if (expression.isInstanceOf[ScalaUDF] && @@ -1049,10 +1050,22 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule queryColumns += getQueryColumn(attr.name, carbonTable) attr + case attr: AggregateExpression => + if (attr.isDistinct) { + isValid = false + } + val aggExp = PreAggregateUtil.validateAggregateFunctionAndGetFields(attr) + if (aggExp.nonEmpty) { + aggreagteExps ++= aggExp + } else { + isValid = false + } + attr + } } } - true + isValid } /**
