HAWQ-964. Ignore Expressions with OR/NOT from partition filtering
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/93052b93 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/93052b93 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/93052b93 Branch: refs/heads/HAWQ-964 Commit: 93052b93098de971eaaa0ed2c5da03372e201509 Parents: f3668dc Author: Kavinder Dhaliwal <[email protected]> Authored: Tue Sep 20 15:35:08 2016 -0700 Committer: Kavinder Dhaliwal <[email protected]> Committed: Tue Sep 20 15:35:08 2016 -0700 ---------------------------------------------------------------------- .../hawq/pxf/plugins/hive/HiveAccessor.java | 86 +++++++++++++------- 1 file changed, 57 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/93052b93/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java index 20a1b9f..2f1c26e 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java @@ -21,6 +21,7 @@ package org.apache.hawq.pxf.plugins.hive; import org.apache.hawq.pxf.api.BasicFilter; import org.apache.hawq.pxf.api.FilterParser; +import org.apache.hawq.pxf.api.LogicalFilter; import org.apache.hawq.pxf.api.utilities.ColumnDescriptor; import org.apache.hawq.pxf.api.utilities.InputData; import org.apache.hawq.pxf.plugins.hdfs.HdfsSplittableDataAccessor; @@ -32,6 +33,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; import java.io.IOException; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -206,51 +208,77 @@ public class HiveAccessor extends HdfsSplittableDataAccessor { return testOneFilter(partitionFields, filter, inputData); } - /* - * We are testing one filter against all the partition fields. The filter - * has the form "fieldA = valueA". The partitions have the form - * partitionOne=valueOne/partitionTwo=ValueTwo/partitionThree=valueThree 1. - * For a filter to match one of the partitions, lets say partitionA for - * example, we need: fieldA = partittionOne and valueA = valueOne. If this - * condition occurs, we return true. 2. If fieldA does not match any one of - * the partition fields we also return true, it means we ignore this filter - * because it is not on a partition field. 3. If fieldA = partittionOne and - * valueA != valueOne, then we return false. - */ - private boolean testOneFilter(List<HivePartition> partitionFields, - Object filter, InputData input) { - // Let's look first at the filter - BasicFilter bFilter = (BasicFilter) filter; + private boolean testForUnsupportedOperators(List<Object> filterList) { + boolean nonAndOp = true; + for (Object filter : filterList) { + if (filter instanceof LogicalFilter) { + if (((LogicalFilter) filter).getOperator() != FilterParser.LogicalOperation.HDOP_AND) + return false; + if (((LogicalFilter) filter).getFilterList() != null) + nonAndOp = testForUnsupportedOperators(((LogicalFilter) filter).getFilterList()); + } + } + return nonAndOp; + } - boolean isFilterOperationEqual = (bFilter.getOperation() == FilterParser.Operation.HDOP_EQ); - if (!isFilterOperationEqual) /* + private boolean testForPartitionEquality(List<HivePartition> partitionFields, List<Object> filterList, InputData input) { + boolean partitionAllowed = true; + for (Object filter : filterList) { + if (filter instanceof BasicFilter) { + BasicFilter bFilter = (BasicFilter) filter; + boolean isFilterOperationEqual = (bFilter.getOperation() == FilterParser.Operation.HDOP_EQ); + if (!isFilterOperationEqual) /* * in case this is not an "equality filter" * we ignore it here - in partition * filtering */{ - return true; - } + return true; + } - int filterColumnIndex = bFilter.getColumn().index(); - String filterValue = bFilter.getConstant().constant().toString(); - ColumnDescriptor filterColumn = input.getColumn(filterColumnIndex); - String filterColumnName = filterColumn.columnName(); + int filterColumnIndex = bFilter.getColumn().index(); + String filterValue = bFilter.getConstant().constant().toString(); + ColumnDescriptor filterColumn = input.getColumn(filterColumnIndex); + String filterColumnName = filterColumn.columnName(); + + for (HivePartition partition : partitionFields) { + if (filterColumnName.equals(partition.name)) { - for (HivePartition partition : partitionFields) { - if (filterColumnName.equals(partition.name)) { /* * the filter field matches a partition field, but the values do * not match */ - return filterValue.equals(partition.val); - } - } + return filterValue.equals(partition.val); + } + } /* * filter field did not match any partition field, so we ignore this * filter and hence return true */ - return true; + } else if (filter instanceof LogicalFilter) { + partitionAllowed = testForPartitionEquality(partitionFields, ((LogicalFilter) filter).getFilterList(), input); + } + } + return partitionAllowed; + } + /* + * We are testing one filter against all the partition fields. The filter + * has the form "fieldA = valueA". The partitions have the form + * partitionOne=valueOne/partitionTwo=ValueTwo/partitionThree=valueThree 1. + * For a filter to match one of the partitions, lets say partitionA for + * example, we need: fieldA = partittionOne and valueA = valueOne. If this + * condition occurs, we return true. 2. If fieldA does not match any one of + * the partition fields we also return true, it means we ignore this filter + * because it is not on a partition field. 3. If fieldA = partittionOne and + * valueA != valueOne, then we return false. + */ + private boolean testOneFilter(List<HivePartition> partitionFields, + Object filter, InputData input) { + // Let's look first at the filter and escape if there are any OR or NOT ops + if (!testForUnsupportedOperators(Arrays.asList(filter))) + return false; + + return testForPartitionEquality(partitionFields, Arrays.asList(filter), input); } private void printOneBasicFilter(Object filter) {
