Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1595#discussion_r155251303
--- Diff:
integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
---
@@ -413,19 +414,35 @@ private[sql] class CarbonLateDecodeStrategy extends
SparkStrategy {
relation: BaseRelation,
predicates: Seq[Expression]): (Seq[Expression], Seq[Filter]) = {
+ def isComplexAttribute(attribute: Attribute) = attribute.dataType
match {
+ case ArrayType(dataType, _) => true
+ case StructType(_) => true
+ case _ => false
+ }
+
+ // In case of ComplexType dataTypes no filters should be pushed down.
IsNotNull is being
+ // explicitly added by spark and pushed. That also has to be handled
and pushed back to
+ // Spark for handling.
+ val predicatesWithoutComplex = predicates.filter(predicate =>
predicate.collect {
+ case a: Attribute if (isComplexAttribute(a)) => a
+ }.size == 0 )
+
// For conciseness, all Catalyst filter expressions of type
`expressions.Expression` below are
// called `predicate`s, while all data source filters of type
`sources.Filter` are simply called
// `filter`s.
val translated: Seq[(Expression, Filter)] =
for {
- predicate <- predicates
+ predicate <- predicatesWithoutComplex
filter <- translateFilter(predicate)
} yield predicate -> filter
+
+ // val newTranslated = translated.toMap.filter()
--- End diff --
remove commented code
---
