Repository: carbondata Updated Branches: refs/heads/master 877efe1a4 -> dbb7ca1b1
[CARBONDATA-3131] Update the requested columns to the Scan Get the updated list of requested columns for Scan task This closes #2952 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dbb7ca1b Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dbb7ca1b Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dbb7ca1b Branch: refs/heads/master Commit: dbb7ca1b17ca647228a5ae04db3e5870af5ad5c8 Parents: 877efe1 Author: dhatchayani <[email protected]> Authored: Tue Nov 27 10:13:33 2018 +0530 Committer: ravipesala <[email protected]> Committed: Tue Nov 27 12:21:40 2018 +0530 ---------------------------------------------------------------------- .../execution/strategy/CarbonLateDecodeStrategy.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/dbb7ca1b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala index 6993ecc..a2c4bd0 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala @@ -390,8 +390,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy { } // Don't request columns that are only referenced by pushed filters. val requestedColumns = - (projectsAttr.to[mutable.LinkedHashSet] ++ filterSet -- handledSet) - .map(relation.attributeMap).toSeq ++ newProjectList + getRequestedColumns(relation, projectsAttr, filterSet, handledSet, newProjectList) var updateRequestedColumns = if (!vectorPushRowFilters && !implictsExisted && !hasDictionaryFilterCols @@ -446,6 +445,15 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy { } } + protected def getRequestedColumns(relation: LogicalRelation, + projectsAttr: Seq[Attribute], + filterSet: AttributeSet, + handledSet: AttributeSet, + newProjectList: Seq[Attribute]) = { + (projectsAttr.to[mutable.LinkedHashSet] ++ filterSet -- handledSet) + .map(relation.attributeMap).toSeq ++ newProjectList + } + private def getDataSourceScan(relation: LogicalRelation, output: Seq[Attribute], partitions: Seq[PartitionSpec],
