Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-963 a14076c2e -> 0f55868a7
HAWQ-963. PXF support for IS_NULL and IS_NOT_NULL conditions Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3f7894df Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3f7894df Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3f7894df Branch: refs/heads/HAWQ-963 Commit: 3f7894df71d0c8f4bf086d747b33073e00daa50d Parents: a14076c Author: Alexander Denissov <[email protected]> Authored: Wed Oct 12 13:46:33 2016 -0700 Committer: Alexander Denissov <[email protected]> Committed: Wed Oct 12 13:46:33 2016 -0700 ---------------------------------------------------------------------- pxf/build.gradle | 4 -- .../org/apache/hawq/pxf/api/FilterParser.java | 41 +++++++++------ .../apache/hawq/pxf/api/FilterParserTest.java | 19 +++++++ .../pxf/plugins/hbase/HBaseFilterBuilder.java | 31 +++++++++++ .../plugins/hbase/HBaseFilterBuilderTest.java | 55 ++++++++++++++++++++ .../pxf/plugins/hive/HiveFilterBuilder.java | 12 +++++ .../hawq/pxf/plugins/hive/HiveORCAccessor.java | 9 +++- .../pxf/plugins/hive/HiveFilterBuilderTest.java | 19 +++++++ 8 files changed, 170 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/build.gradle ---------------------------------------------------------------------- diff --git a/pxf/build.gradle b/pxf/build.gradle index a7f4610..8c37dfc 100644 --- a/pxf/build.gradle +++ b/pxf/build.gradle @@ -323,10 +323,6 @@ project('pxf-hive') { compile "org.apache.hive:hive-common:$hiveVersion" compile "org.apache.hive:hive-serde:$hiveVersion" testCompile 'pl.pragmatists:JUnitParams:1.0.2' - configurations { - // Remove hive-exec from unit tests as it causes VerifyError - testRuntime.exclude module: 'hive-exec' - } } ospackage { http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java index 22c76a6..da10ced 100644 --- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java @@ -63,7 +63,9 @@ public class FilterParser { HDOP_EQ, HDOP_NE, HDOP_AND, - HDOP_LIKE + HDOP_LIKE, + HDOP_ISNULL, + HDOP_ISNOTNULL } public enum LogicalOperation { @@ -88,6 +90,7 @@ public class FilterParser { * @throws Exception if building the filter failed */ public Object build(Operation operation, Object left, Object right) throws Exception; + public Object build(Operation operation, Object operand) throws Exception; public Object build(LogicalOperation operation, Object left, Object right) throws Exception; public Object build(LogicalOperation operation, Object filter) throws Exception; } @@ -176,20 +179,26 @@ public class FilterParser { } Object rightOperand = operandsStack.pop(); - // Pop left operand - if (operandsStack.empty()) { - throw new FilterStringSyntaxException("missing operands for op " + operation + " at " + index); + // all operations other than null checks require 2 operands + Object result; + if (operation == Operation.HDOP_ISNULL || operation == Operation.HDOP_ISNOTNULL) { + result = filterBuilder.build(operation, rightOperand); + } + else { + // Pop left operand + if (operandsStack.empty()) { + throw new FilterStringSyntaxException("missing operands for op " + operation + " at " + index); + } + Object leftOperand = operandsStack.pop(); + + // Normalize order, evaluate + // Column should be on the left + result = (leftOperand instanceof Constant) + // column on the right, reverse expression + ? filterBuilder.build(reverseOp(operation), rightOperand, leftOperand) + // no swap, column on the left + : filterBuilder.build(operation, leftOperand, rightOperand); } - Object leftOperand = operandsStack.pop(); - - // Normalize order, evaluate - // Column should be on the left - Object result = (leftOperand instanceof Constant) - // column on the right, reverse expression - ? filterBuilder.build(reverseOp(operation), rightOperand, leftOperand) - // no swap, column on the left - : filterBuilder.build(operation, leftOperand, rightOperand); - // Store result on stack operandsStack.push(result); break; @@ -205,7 +214,7 @@ public class FilterParser { result = filterBuilder.build(logicalOperation, exp); } else { rightOperand = operandsStack.pop(); - leftOperand = operandsStack.pop(); + Object leftOperand = operandsStack.pop(); result = filterBuilder.build(logicalOperation, leftOperand, rightOperand); } @@ -385,6 +394,8 @@ public class FilterParser { operatorTranslationMap.put(6, Operation.HDOP_NE); operatorTranslationMap.put(7, Operation.HDOP_AND); operatorTranslationMap.put(8, Operation.HDOP_LIKE); + operatorTranslationMap.put(9, Operation.HDOP_ISNULL); + operatorTranslationMap.put(10, Operation.HDOP_ISNOTNULL); return operatorTranslationMap; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java index a129a4b..fc2df4c 100644 --- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java +++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java @@ -223,6 +223,15 @@ public class FilterParserTest { filter = "a1c2o8"; op = Operation.HDOP_LIKE; runParseOneOperation("this filter was built from HDOP_LIKE", filter, op); + + filter = "a1o9"; + op = Operation.HDOP_ISNULL; + runParseOneUnaryOperation("this filter was build from HDOP_ISNULL", filter, op); + + filter = "a1o10"; + op = Operation.HDOP_ISNOTNULL; + runParseOneUnaryOperation("this filter was build from HDOP_ISNOTNULL", filter, op); + } @Test @@ -259,6 +268,7 @@ public class FilterParserTest { filter = "c2a1o8"; op = Operation.HDOP_LIKE; runParseOneOperation("this filter was build from HDOP_LIKE using reverse!", filter, op); + } @Test @@ -420,6 +430,15 @@ public class FilterParserTest { assertEquals(description, result); } + private void runParseOneUnaryOperation(String description, String filter, Operation op) throws Exception { + when(filterBuilder.build(eq(op), + any())).thenReturn(description); + + Object result = filterParser.parse(filter); + + assertEquals(description, result); + } + private String filterStringMsg(String filter) { return " (filter string: '" + filter + "')"; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java index 5ec0652..b84c21f 100644 --- a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java +++ b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java @@ -138,6 +138,11 @@ public class HBaseFilterBuilder implements FilterParser.FilterBuilder { } @Override + public Object build(FilterParser.Operation operation, Object operand) throws Exception { + return handleSimpleOperations(operation, (FilterParser.ColumnIndex) operand); + } + + @Override public Object build(FilterParser.LogicalOperation opId, Object leftOperand, Object rightOperand) { return handleCompoundOperations(opId, (Filter) leftOperand, (Filter) rightOperand); } @@ -167,6 +172,32 @@ public class HBaseFilterBuilder implements FilterParser.FilterBuilder { } /** + * Handles simple column-operator expressions. + */ + private Filter handleSimpleOperations(FilterParser.Operation opId, + FilterParser.ColumnIndex column) throws Exception { + HBaseColumnDescriptor hbaseColumn = tupleDescription.getColumn(column.index()); + CompareFilter.CompareOp compareOperation; + ByteArrayComparable comparator; + switch (opId) { + case HDOP_ISNULL: + compareOperation = CompareFilter.CompareOp.EQUAL; + comparator = new NullComparator(); + break; + case HDOP_ISNOTNULL: + compareOperation = CompareFilter.CompareOp.NOT_EQUAL; + comparator = new NullComparator(); + break; + default: + throw new Exception("unsupported unary operation for filtering " + opId); + } + return new SingleColumnValueFilter(hbaseColumn.columnFamilyBytes(), + hbaseColumn.qualifierBytes(), + compareOperation, + comparator); + } + + /** * Handles simple column-operator-constant expressions. * Creates a special filter in the case the column is the row key column. */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java b/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java index 9bc5474..0d9de3d 100644 --- a/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java +++ b/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java @@ -1,8 +1,15 @@ package org.apache.hawq.pxf.plugins.hbase; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.NullComparator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hawq.pxf.plugins.hbase.utilities.HBaseColumnDescriptor; +import org.apache.hawq.pxf.plugins.hbase.utilities.HBaseTupleDescription; import org.junit.Test; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class HBaseFilterBuilderTest { @@ -13,4 +20,52 @@ public class HBaseFilterBuilderTest { assertNull(builder.getFilterObject(filter)); } + @Test + public void parseISNULLExpression() throws Exception { + String filter = "a1o9"; + HBaseTupleDescription desc = mock(HBaseTupleDescription.class); + HBaseColumnDescriptor column = mock(HBaseColumnDescriptor.class); + when(desc.getColumn(1)).thenReturn(column); + + byte[] family = new byte[]{}; + byte[] qualifier = new byte[]{}; + + when(column.columnFamilyBytes()).thenReturn(family); + when(column.qualifierBytes()).thenReturn(qualifier); + + HBaseFilterBuilder builder = new HBaseFilterBuilder(desc); + SingleColumnValueFilter result = (SingleColumnValueFilter) builder.getFilterObject(filter); + + assertNotNull(result); + assertSame(family, result.getFamily()); + assertSame(qualifier, result.getQualifier()); + assertEquals(CompareFilter.CompareOp.EQUAL, result.getOperator()); + assertTrue(result.getComparator() instanceof NullComparator); + + } + + @Test + public void parseISNOTNULLExpression() throws Exception { + String filter = "a1o10"; + HBaseTupleDescription desc = mock(HBaseTupleDescription.class); + HBaseColumnDescriptor column = mock(HBaseColumnDescriptor.class); + when(desc.getColumn(1)).thenReturn(column); + + byte[] family = new byte[]{}; + byte[] qualifier = new byte[]{}; + + when(column.columnFamilyBytes()).thenReturn(family); + when(column.qualifierBytes()).thenReturn(qualifier); + + HBaseFilterBuilder builder = new HBaseFilterBuilder(desc); + SingleColumnValueFilter result = (SingleColumnValueFilter) builder.getFilterObject(filter); + + assertNotNull(result); + assertSame(family, result.getFamily()); + assertSame(qualifier, result.getQualifier()); + assertEquals(CompareFilter.CompareOp.NOT_EQUAL, result.getOperator()); + assertTrue(result.getComparator() instanceof NullComparator); + + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java index bd82a3b..6d76036 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java @@ -93,6 +93,18 @@ public class HiveFilterBuilder implements FilterParser.FilterBuilder { (FilterParser.Constant) rightOperand); } + @Override + public Object build(FilterParser.Operation operation, Object operand) throws Exception { + if (operation == FilterParser.Operation.HDOP_ISNULL || operation == FilterParser.Operation.HDOP_ISNOTNULL) { + // use null for the constant value of null comparison + return handleSimpleOperations(operation, + (FilterParser.ColumnIndex) operand, + null); + } else { + throw new Exception("Unsupported unary operation " + operation); + } + } + /* * Handles simple column-operator-constant expressions Creates a special * filter in the case the column is the row key column http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java index ab2f96e..72ab4e1 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java @@ -143,7 +143,8 @@ public class HiveORCAccessor extends HiveAccessor { /* The below functions will not be compatible and requires update with Hive 2.0 APIs */ BasicFilter filter = (BasicFilter) filterObj; int filterColumnIndex = filter.getColumn().index(); - Object filterValue = filter.getConstant().constant(); + // filter value might be null for unary operations + Object filterValue = filter.getConstant() == null ? null : filter.getConstant().constant(); ColumnDescriptor filterColumn = inputData.getColumn(filterColumnIndex); String filterColumnName = filterColumn.columnName(); @@ -166,6 +167,12 @@ public class HiveORCAccessor extends HiveAccessor { case HDOP_NE: builder.startNot().equals(filterColumnName, filterValue).end(); break; + case HDOP_ISNULL: + builder.isNull(filterColumnName); + break; + case HDOP_ISNOTNULL: + builder.startNot().isNull(filterColumnName).end(); + break; } return; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java index e0e6536..7954cd0 100755 --- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java +++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java @@ -29,6 +29,7 @@ import org.apache.hawq.pxf.api.BasicFilter; import static org.apache.hawq.pxf.api.FilterParser.Operation; import static org.apache.hawq.pxf.api.FilterParser.Operation.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class HiveFilterBuilderTest { @Test @@ -63,4 +64,22 @@ public class HiveFilterBuilderTest { assertEquals(HDOP_LT, ((BasicFilter) filter.getFilterList().get(1)).getOperation()); } + @Test + public void parseISNULLExpression() throws Exception { + HiveFilterBuilder builder = new HiveFilterBuilder(null); + BasicFilter filter = (BasicFilter) builder.getFilterObject("a1o9"); + assertEquals(Operation.HDOP_ISNULL, filter.getOperation()); + assertEquals(1, filter.getColumn().index()); + assertNull(filter.getConstant()); + } + + @Test + public void parseISNOTNULLExpression() throws Exception { + HiveFilterBuilder builder = new HiveFilterBuilder(null); + BasicFilter filter = (BasicFilter) builder.getFilterObject("a1o10"); + assertEquals(Operation.HDOP_ISNOTNULL, filter.getOperation()); + assertEquals(1, filter.getColumn().index()); + assertNull(filter.getConstant()); + } + }
