This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit a838531f243f4b826778b51d1f238461d3414fe4 Author: Indhumathi27 <[email protected]> AuthorDate: Fri Mar 4 13:25:36 2022 +0530 [CARBONDATA-4306] Fix Query Performance issue for Spark 3.1 Why is this PR needed? Some non-partition filters, which cannot be handled by carbon, is not pushed down to spark. What changes were proposed in this PR? If partition filters is non empty, then the filter column is not partition column, then push the filter to spark This closes #4252 --- .../apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala index 824c7fb..85044a2 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala @@ -158,10 +158,10 @@ private[sql] object CarbonSourceStrategy extends SparkStrategy { SparkSession.getActiveSession.get, relation.catalogTable.get.identifier ) + // remove dynamic partition filter from predicates + filterPredicates = CarbonToSparkAdapter.getDataFilter(partitionSet, + allPredicates, partitionsFilter) } - // remove dynamic partition filter from predicates - filterPredicates = CarbonToSparkAdapter.getDataFilter(partitionSet, - allPredicates, partitionsFilter) val table = relation.relation.asInstanceOf[CarbonDatasourceHadoopRelation] val projects = rawProjects.map {p => p.transform {
