Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2818#discussion_r226324001
--- Diff:
integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
---
@@ -337,19 +340,35 @@ private[sql] class CarbonLateDecodeStrategy extends
SparkStrategy {
metadata,
needDecoder,
updateRequestedColumns.asInstanceOf[Seq[Attribute]])
- filterCondition.map(execution.FilterExec(_, scan)).getOrElse(scan)
+ // Check whether spark should handle row filters in case of vector
flow.
+ if (!vectorPushRowFilters && scan.isInstanceOf[CarbonDataSourceScan]
+ && !hasDictionaryFilterCols) {
+ // Here carbon only do page pruning and row level pruning will be
done by spark.
+ scan.inputRDDs().head match {
+ case rdd: CarbonScanRDD[InternalRow] =>
+ rdd.setDirectScanSupport(true)
+ case _ =>
+ }
+
filterPredicates.reduceLeftOption(expressions.And).map(execution.FilterExec(_,
scan))
+ .getOrElse(scan)
+ } else {
+ filterCondition.map(execution.FilterExec(_, scan)).getOrElse(scan)
+ }
} else {
var newProjectList: Seq[Attribute] = Seq.empty
+ var implictsExisted = false
--- End diff --
why this is required, can you add comment
---
