kunal642 commented on a change in pull request #3861: URL: https://github.com/apache/carbondata/pull/3861#discussion_r481170533
########## File path: integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonSecondaryIndexOptimizer.scala ########## @@ -684,21 +730,71 @@ class CarbonSecondaryIndexOptimizer(sparkSession: SparkSession) { .getBoolean("spark.carbon.pushdown.join.as.filter", defaultValue = true) val transformChild = false var addProjection = needProjection + // to store the sort node per query + var sortNodeForPushDown: Sort = null + // to store the limit literal per query + var limitLiteral: Literal = null + // by default do not push down notNull filter, + // but for orderby limit push down, push down notNull filter also. Else we get wrong results. + var pushDownNotNullFilter: Boolean = false val transformedPlan = transformPlan(plan, { - case union@Union(children) => + case union@Union(_) => // In case of Union, Extra Project has to be added to the Plan. Because if left table is // pushed to SI and right table is not pushed, then Output Attribute mismatch will happen addProjection = true (union, true) - case sort@Sort(order, global, plan) => + case sort@Sort(_, _, _) => addProjection = true (sort, true) - case filter@Filter(condition, logicalRelation@MatchIndexableRelation(indexableRelation)) + case limit@Limit(literal: Literal, sort@Sort(_, _, child)) => + child match { + case filter: Filter => + if (checkIfPushDownOrderByLimitAndNotNullFilter(literal, sort, filter)) { + sortNodeForPushDown = sort + limitLiteral = literal + pushDownNotNullFilter = true + } + case p: Project if (p.child.isInstanceOf[Filter]) => + if (checkIfPushDownOrderByLimitAndNotNullFilter(literal, + sort, + p.child.asInstanceOf[Filter])) { + sortNodeForPushDown = sort + limitLiteral = literal + pushDownNotNullFilter = true + } + case _ => + } + (limit, transformChild) + case limit@Limit(literal: Literal, _@Project(_, child)) if child.isInstanceOf[Sort] => Review comment: if you use the following then you will not have to check for isInstanceOf of cast the child to Sort. `case limit@Limit(literal: Literal, _@Project(_, Sort(_, _)))` ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org