Repository: carbondata Updated Branches: refs/heads/master 9c83bd18e -> f70f23049
[CARBONDATA-1279] Push down for queries like %xxx , ends with are not working as expected in Spark 2.1 This closes #1279 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f70f2304 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f70f2304 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f70f2304 Branch: refs/heads/master Commit: f70f230493535994919126f60fb3461b57be2447 Parents: 9c83bd1 Author: root <[email protected]> Authored: Mon Aug 21 08:39:46 2017 +0000 Committer: Venkata Ramana G <[email protected]> Committed: Wed Aug 30 10:17:14 2017 +0530 ---------------------------------------------------------------------- .../apache/spark/sql/CarbonBoundReference.scala | 8 ++++++++ .../sql/execution/CarbonLateDecodeStrategy.scala | 4 ++++ .../spark/sql/optimizer/CarbonFilters.scala | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/f70f2304/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala index eb8a7a0..6f578ac 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala @@ -49,3 +49,11 @@ case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nu override def newInstance(): NamedExpression = throw new UnsupportedOperationException } +case class CarbonEndsWith(expr: Expression) extends Filter { + override def references: Array[String] = null +} + +case class CarbonContainsWith(expr: Expression) extends Filter { + override def references: Array[String] = null +} + http://git-wip-us.apache.org/repos/asf/carbondata/blob/f70f2304/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala index 33bba8f..eac0a28 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala @@ -512,6 +512,10 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy { CastExpressionOptimization.checkIfCastCanBeRemove(c) case StartsWith(a: Attribute, Literal(v, t)) => Some(sources.StringStartsWith(a.name, v.toString)) + case c@EndsWith(a: Attribute, Literal(v, t)) => + Some(CarbonEndsWith(c)) + case c@Contains(a: Attribute, Literal(v, t)) => + Some(CarbonContainsWith(c)) case others => None } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f70f2304/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala index 1bc97d8..31a3208 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.CarbonBoundReference import org.apache.spark.sql.CastExpr import org.apache.spark.sql.sources import org.apache.spark.sql.types._ +import org.apache.spark.sql.CarbonContainsWith +import org.apache.spark.sql.CarbonEndsWith import org.apache.carbondata.core.metadata.datatype.DataType import org.apache.carbondata.core.metadata.schema.table.CarbonTable @@ -105,6 +107,18 @@ object CarbonFilters { val r = new LessThanExpression( getCarbonExpression(name), getCarbonLiteralExpression(name, maxValueLimit)) Some(new AndExpression(l, r)) + case CarbonEndsWith(expr: Expression) => + Some(new SparkUnknownExpression(expr.transform { + case AttributeReference(name, dataType, _, _) => + CarbonBoundReference(new CarbonColumnExpression(name.toString, + CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable) + })) + case CarbonContainsWith(expr: Expression) => + Some(new SparkUnknownExpression(expr.transform { + case AttributeReference(name, dataType, _, _) => + CarbonBoundReference(new CarbonColumnExpression(name.toString, + CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable) + })) case CastExpr(expr: Expression) => Some(transformExpression(expr)) case _ => None @@ -235,6 +249,10 @@ object CarbonFilters { CastExpressionOptimization.checkIfCastCanBeRemove(c) case StartsWith(a: Attribute, Literal(v, t)) => Some(sources.StringStartsWith(a.name, v.toString)) + case c@EndsWith(a: Attribute, Literal(v, t)) => + Some(CarbonEndsWith(c)) + case c@Contains(a: Attribute, Literal(v, t)) => + Some(CarbonContainsWith(c)) case c@Cast(a: Attribute, _) => Some(CastExpr(c)) case others =>
