Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2439#discussion_r200281974
--- Diff:
integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
---
@@ -69,84 +69,90 @@ case class CarbonDatasourceHadoopRelation(
override def schema: StructType =
tableSchema.getOrElse(carbonRelation.schema)
def buildScan(requiredColumns: Array[String],
+ filterComplex:
Seq[org.apache.spark.sql.catalyst.expressions.Expression],
projects: Seq[NamedExpression],
filters: Array[Filter],
partitions: Seq[PartitionSpec]): RDD[InternalRow] = {
val filterExpression: Option[Expression] = filters.flatMap { filter =>
CarbonFilters.createCarbonFilter(schema, filter)
}.reduceOption(new AndExpression(_, _))
- var parentColumn = new ListBuffer[String]
- // In case of Struct or StructofStruct Complex type, get the project
column for given
- // parent/child field and pushdown the corresponding project column.
In case of Array,
- // ArrayofStruct or StructofArray, pushdown parent column
- var reqColumns = projects.map {
- case a@Alias(s: GetStructField, name) =>
- var arrayTypeExists = false
- var ifGetArrayItemExists = s
- breakable({
- while (ifGetArrayItemExists.containsChild != null) {
- if
(ifGetArrayItemExists.child.isInstanceOf[AttributeReference]) {
- arrayTypeExists =
s.childSchema.toString().contains("ArrayType")
- break
- } else {
- if (ifGetArrayItemExists.child.isInstanceOf[GetArrayItem]) {
- arrayTypeExists = true
+ val projection = new CarbonProjection
+
+ val complexFilterExists = filterComplex.map(col =>
+ col.map(_.isInstanceOf[GetStructField]))
+
+ if (!complexFilterExists.exists(f => f.contains(true))) {
--- End diff --
add detailed comment on why this check was needed
---
