kunal642 commented on a change in pull request #3861:
URL: https://github.com/apache/carbondata/pull/3861#discussion_r481170533
##########
File path:
integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonSecondaryIndexOptimizer.scala
##########
@@ -684,21 +730,71 @@ class CarbonSecondaryIndexOptimizer(sparkSession:
SparkSession) {
.getBoolean("spark.carbon.pushdown.join.as.filter", defaultValue = true)
val transformChild = false
var addProjection = needProjection
+ // to store the sort node per query
+ var sortNodeForPushDown: Sort = null
+ // to store the limit literal per query
+ var limitLiteral: Literal = null
+ // by default do not push down notNull filter,
+ // but for orderby limit push down, push down notNull filter also. Else we
get wrong results.
+ var pushDownNotNullFilter: Boolean = false
val transformedPlan = transformPlan(plan, {
- case union@Union(children) =>
+ case union@Union(_) =>
// In case of Union, Extra Project has to be added to the Plan.
Because if left table is
// pushed to SI and right table is not pushed, then Output Attribute
mismatch will happen
addProjection = true
(union, true)
- case sort@Sort(order, global, plan) =>
+ case sort@Sort(_, _, _) =>
addProjection = true
(sort, true)
- case filter@Filter(condition,
logicalRelation@MatchIndexableRelation(indexableRelation))
+ case limit@Limit(literal: Literal, sort@Sort(_, _, child)) =>
+ child match {
+ case filter: Filter =>
+ if (checkIfPushDownOrderByLimitAndNotNullFilter(literal, sort,
filter)) {
+ sortNodeForPushDown = sort
+ limitLiteral = literal
+ pushDownNotNullFilter = true
+ }
+ case p: Project if (p.child.isInstanceOf[Filter]) =>
+ if (checkIfPushDownOrderByLimitAndNotNullFilter(literal,
+ sort,
+ p.child.asInstanceOf[Filter])) {
+ sortNodeForPushDown = sort
+ limitLiteral = literal
+ pushDownNotNullFilter = true
+ }
+ case _ =>
+ }
+ (limit, transformChild)
+ case limit@Limit(literal: Literal, _@Project(_, child)) if
child.isInstanceOf[Sort] =>
Review comment:
if you use the following then you will not have to check for
isInstanceOf of cast the child to Sort.
`case limit@Limit(literal: Literal, _@Project(_, Sort(_, _)))`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
