[
https://issues.apache.org/jira/browse/KYLIN-4980?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17331212#comment-17331212
]
ASF GitHub Bot commented on KYLIN-4980:
---------------------------------------
zhengshengjun commented on a change in pull request #1642:
URL: https://github.com/apache/kylin/pull/1642#discussion_r619646558
##########
File path:
kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
##########
@@ -295,8 +295,48 @@ class FilePruner(cubeInstance: CubeInstance,
}
}
- private def getSpecFilter(dataFilters: Seq[Expression], col: Attribute):
Seq[Expression] = {
- dataFilters.filter(_.references.subsetOf(AttributeSet(col)))
+ private def getSegmentFilter(dataFilters: Seq[Expression], col: Attribute):
Seq[Expression] = {
+ dataFilters.map(extractSegmentFilter(_,
col)).filter(!_.equals(None)).map(_.get)
+ }
+
+ private def extractSegmentFilter(filter: Expression, col: Attribute):
Option[Expression] = {
+ filter match {
+ case expressions.Or(left, right) =>
+ val leftChild = extractSegmentFilter(left, col)
+ val rightChild = extractSegmentFilter(right, col)
+
+ //if there exists leaf-node that doesn't contain partition column, the
parent filter is
+ //unnecessary for segment prunning.
+ //e.g. "where a = xxx or partition = xxx", we can't filter any segment
+ if (leftChild.eq(None) || rightChild.eq(None)) {
+ None
+ } else {
+ Some(expressions.Or(leftChild.get, rightChild.get))
+ }
+ case expressions.And(left, right) =>
+ val leftChild = extractSegmentFilter(left, col)
+ val rightChild = extractSegmentFilter(right, col)
+
+ //if there is only one leaf-node that contains partition column
+ //e.g. "where a = xxx and partition = xxx",
+ //then we can filter segment using "where partition = xxx"
+ if (!leftChild.eq(None) && !rightChild.eq(None)) {
+ Some(expressions.And(leftChild.get, rightChild.get))
+ } else if (!rightChild.eq(None)) {
+ rightChild
+ } else if (!leftChild.eq(None)) {
+ leftChild
+ } else {
+ None
+ }
+ case _ =>
+ //other unary filter like EqualTo, GreaterThan, GreaterThanOrEqual,
etc.
+ if (filter.references.contains(col)) {
Review comment:
good suggestion ~
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Support prunning segments from complex filter conditions
> --------------------------------------------------------
>
> Key: KYLIN-4980
> URL: https://issues.apache.org/jira/browse/KYLIN-4980
> Project: Kylin
> Issue Type: Improvement
> Components: Query Engine
> Affects Versions: v4.0.0-beta
> Reporter: ShengJun Zheng
> Assignee: ShengJun Zheng
> Priority: Major
> Fix For: v4.0.0-GA
>
>
> Segment pruner can't prune segment from complex filter conditions, like the
> filter condition below:
> "where (col_a = xxx and col_partition = xxx) or (col_b=xxx and col_partition
> = xxx)"
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
