Repository: carbondata Updated Branches: refs/heads/master 12c28c946 -> 133ec17e5
[CARBONDATA-2606][Complex DataType Enhancements]Fix for ComplexDataType Projection PushDown Problem1: Fix for ComplexDataType Projection PushDown when Table Schema contains ColumnName in UpperCase Solution: Change ColumnName to Lowercase Problem2: If Struct contains Array, pushdown only parent column Solution: Check for ArrayType or GetArrayItem in the Complex Column, if any ArrayType is found, then pushdown parent column This closes #2421 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/133ec17e Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/133ec17e Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/133ec17e Branch: refs/heads/master Commit: 133ec17e527c22d8de1353f723fb6d77cd404eeb Parents: 12c28c9 Author: Indhumathi27 <[email protected]> Authored: Wed Jun 27 13:01:28 2018 +0530 Committer: kunal642 <[email protected]> Committed: Tue Jul 3 14:49:31 2018 +0530 ---------------------------------------------------------------------- .../core/scan/model/QueryModelBuilder.java | 6 ++-- .../sql/CarbonDatasourceHadoopRelation.scala | 37 +++++++++++++------- 2 files changed, 29 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/133ec17e/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java index c75f2bc..ced80f2 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java @@ -82,13 +82,15 @@ public class QueryModelBuilder { } } projection = optimizeProjectionForComplexColumns(projection, projectionColumns, factTableName); + List<String> projectionDimensionAndMeasures = new ArrayList<>(); this.projection = projection; for (ProjectionDimension projectionDimension : projection.getDimensions()) { - LOGGER.info("Project Columns: " + projectionDimension.getColumnName()); + projectionDimensionAndMeasures.add(projectionDimension.getColumnName()); } for (ProjectionMeasure projectionMeasure : projection.getMeasures()) { - LOGGER.info("Project Columns: " + projectionMeasure.getColumnName()); + projectionDimensionAndMeasures.add(projectionMeasure.getColumnName()); } + LOGGER.info("Projection Columns: " + projectionDimensionAndMeasures); return this; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/133ec17e/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala index fad944c..86e9ace 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import scala.util.control.Breaks._ import org.apache.spark.CarbonInputMetrics import org.apache.spark.rdd.RDD @@ -75,24 +76,34 @@ case class CarbonDatasourceHadoopRelation( CarbonFilters.createCarbonFilter(schema, filter) }.reduceOption(new AndExpression(_, _)) + var parentColumn = new ListBuffer[String] // In case of Struct or StructofStruct Complex type, get the project column for given // parent/child field and pushdown the corresponding project column. In case of Array, // ArrayofStruct or StructofArray, pushdown parent column var reqColumns = projects.map { case a@Alias(s: GetStructField, name) => - val arrayTypeExists = s.childSchema.map(x => x.dataType) - .filter(dataType => dataType.isInstanceOf[ArrayType]) - val ifGetArrayItem = s.child.map(x => x.isInstanceOf[GetArrayItem]) - var ifGetArrayItemExists = false - ifGetArrayItem.foreach(ifexists => - if (ifexists.equals(true)) { - ifGetArrayItemExists = true + var arrayTypeExists = false + var ifGetArrayItemExists = s + breakable({ + while (ifGetArrayItemExists.containsChild != null) { + if (ifGetArrayItemExists.child.isInstanceOf[AttributeReference]) { + arrayTypeExists = s.childSchema.toString().contains("ArrayType") + break + } else { + if (ifGetArrayItemExists.child.isInstanceOf[GetArrayItem]) { + arrayTypeExists = true + break + } + else { + ifGetArrayItemExists = ifGetArrayItemExists.child.asInstanceOf[GetStructField] + } + } } - ) - if (0 == arrayTypeExists.length && ifGetArrayItemExists.equals(false)) { + }) + if (!arrayTypeExists) { + parentColumn += s.toString().split("\\.")(0).replaceAll("#.*", "").toLowerCase + parentColumn = parentColumn.distinct s.toString().replaceAll("#[0-9]*", "").toLowerCase - } else if (ifGetArrayItemExists.equals(true)) { - s.toString().split("\\.")(0).replaceAll("#.*", "").toLowerCase } else { s.toString().split("\\.")(0).replaceAll("#.*", "").toLowerCase } @@ -117,7 +128,9 @@ case class CarbonDatasourceHadoopRelation( if (null != reqColumns && reqColumns.nonEmpty) { reqColumns.foreach(reqCol => { - if (!reqCol.toString.equalsIgnoreCase(col) && !reqCol.toString.startsWith(col + ".")) { + if (!reqCol.toString.equalsIgnoreCase(col) && + !reqCol.toString.startsWith(col.toLowerCase + ".") && + !parentColumn.contains(col.toLowerCase)) { output += col } else { output += reqCol.toString
