Repository: incubator-hawq
Updated Branches:
  refs/heads/HAWQ-963 a14076c2e -> 0f55868a7


HAWQ-963. PXF support for IS_NULL and IS_NOT_NULL conditions


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3f7894df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3f7894df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3f7894df

Branch: refs/heads/HAWQ-963
Commit: 3f7894df71d0c8f4bf086d747b33073e00daa50d
Parents: a14076c
Author: Alexander Denissov <adenis...@pivotal.io>
Authored: Wed Oct 12 13:46:33 2016 -0700
Committer: Alexander Denissov <adenis...@pivotal.io>
Committed: Wed Oct 12 13:46:33 2016 -0700

----------------------------------------------------------------------
 pxf/build.gradle                                |  4 --
 .../org/apache/hawq/pxf/api/FilterParser.java   | 41 +++++++++------
 .../apache/hawq/pxf/api/FilterParserTest.java   | 19 +++++++
 .../pxf/plugins/hbase/HBaseFilterBuilder.java   | 31 +++++++++++
 .../plugins/hbase/HBaseFilterBuilderTest.java   | 55 ++++++++++++++++++++
 .../pxf/plugins/hive/HiveFilterBuilder.java     | 12 +++++
 .../hawq/pxf/plugins/hive/HiveORCAccessor.java  |  9 +++-
 .../pxf/plugins/hive/HiveFilterBuilderTest.java | 19 +++++++
 8 files changed, 170 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/build.gradle
----------------------------------------------------------------------
diff --git a/pxf/build.gradle b/pxf/build.gradle
index a7f4610..8c37dfc 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -323,10 +323,6 @@ project('pxf-hive') {
         compile "org.apache.hive:hive-common:$hiveVersion"
         compile "org.apache.hive:hive-serde:$hiveVersion"
         testCompile 'pl.pragmatists:JUnitParams:1.0.2'
-        configurations {
-            // Remove hive-exec from unit tests as it causes VerifyError
-            testRuntime.exclude module: 'hive-exec'
-        }
     }
 
     ospackage {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
----------------------------------------------------------------------
diff --git 
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java 
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
index 22c76a6..da10ced 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
@@ -63,7 +63,9 @@ public class FilterParser {
         HDOP_EQ,
         HDOP_NE,
         HDOP_AND,
-        HDOP_LIKE
+        HDOP_LIKE,
+        HDOP_ISNULL,
+        HDOP_ISNOTNULL
     }
 
     public enum LogicalOperation {
@@ -88,6 +90,7 @@ public class FilterParser {
          * @throws Exception if building the filter failed
          */
         public Object build(Operation operation, Object left, Object right) 
throws Exception;
+        public Object build(Operation operation, Object operand) throws 
Exception;
         public Object build(LogicalOperation operation, Object left, Object 
right) throws Exception;
         public Object build(LogicalOperation operation, Object filter) throws 
Exception;
     }
@@ -176,20 +179,26 @@ public class FilterParser {
                     }
                     Object rightOperand = operandsStack.pop();
 
-                    // Pop left operand
-                    if (operandsStack.empty()) {
-                        throw new FilterStringSyntaxException("missing 
operands for op " + operation + " at " + index);
+                    // all operations other than null checks require 2 operands
+                    Object result;
+                    if (operation == Operation.HDOP_ISNULL || operation == 
Operation.HDOP_ISNOTNULL) {
+                        result = filterBuilder.build(operation, rightOperand);
+                    }
+                    else {
+                        // Pop left operand
+                        if (operandsStack.empty()) {
+                            throw new FilterStringSyntaxException("missing 
operands for op " + operation + " at " + index);
+                        }
+                        Object leftOperand = operandsStack.pop();
+
+                        // Normalize order, evaluate
+                        // Column should be on the left
+                        result = (leftOperand instanceof Constant)
+                                // column on the right, reverse expression
+                                ? filterBuilder.build(reverseOp(operation), 
rightOperand, leftOperand)
+                                // no swap, column on the left
+                                : filterBuilder.build(operation, leftOperand, 
rightOperand);
                     }
-                    Object leftOperand = operandsStack.pop();
-
-                    // Normalize order, evaluate
-                    // Column should be on the left
-                    Object result = (leftOperand instanceof Constant)
-                            // column on the right, reverse expression
-                            ? filterBuilder.build(reverseOp(operation), 
rightOperand, leftOperand)
-                            // no swap, column on the left
-                            : filterBuilder.build(operation, leftOperand, 
rightOperand);
-
                     // Store result on stack
                     operandsStack.push(result);
                     break;
@@ -205,7 +214,7 @@ public class FilterParser {
                         result = filterBuilder.build(logicalOperation, exp);
                     } else {
                         rightOperand  = operandsStack.pop();
-                        leftOperand = operandsStack.pop();
+                        Object leftOperand = operandsStack.pop();
 
                         result = filterBuilder.build(logicalOperation, 
leftOperand, rightOperand);
                     }
@@ -385,6 +394,8 @@ public class FilterParser {
         operatorTranslationMap.put(6, Operation.HDOP_NE);
         operatorTranslationMap.put(7, Operation.HDOP_AND);
         operatorTranslationMap.put(8, Operation.HDOP_LIKE);
+        operatorTranslationMap.put(9, Operation.HDOP_ISNULL);
+        operatorTranslationMap.put(10, Operation.HDOP_ISNOTNULL);
         return operatorTranslationMap;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
----------------------------------------------------------------------
diff --git 
a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java 
b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
index a129a4b..fc2df4c 100644
--- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
@@ -223,6 +223,15 @@ public class FilterParserTest {
         filter = "a1c2o8";
         op = Operation.HDOP_LIKE;
         runParseOneOperation("this filter was built from HDOP_LIKE", filter, 
op);
+
+        filter = "a1o9";
+        op = Operation.HDOP_ISNULL;
+        runParseOneUnaryOperation("this filter was build from HDOP_ISNULL", 
filter, op);
+
+        filter = "a1o10";
+        op = Operation.HDOP_ISNOTNULL;
+        runParseOneUnaryOperation("this filter was build from HDOP_ISNOTNULL", 
filter, op);
+
     }
 
     @Test
@@ -259,6 +268,7 @@ public class FilterParserTest {
         filter = "c2a1o8";
         op = Operation.HDOP_LIKE;
         runParseOneOperation("this filter was build from HDOP_LIKE using 
reverse!", filter, op);
+
     }
 
     @Test
@@ -420,6 +430,15 @@ public class FilterParserTest {
         assertEquals(description, result);
     }
 
+    private void runParseOneUnaryOperation(String description, String filter, 
Operation op) throws Exception {
+        when(filterBuilder.build(eq(op),
+                any())).thenReturn(description);
+
+        Object result = filterParser.parse(filter);
+
+        assertEquals(description, result);
+    }
+
     private String filterStringMsg(String filter) {
         return " (filter string: '" + filter + "')";
     }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
----------------------------------------------------------------------
diff --git 
a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
 
b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
index 5ec0652..b84c21f 100644
--- 
a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
+++ 
b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
@@ -138,6 +138,11 @@ public class HBaseFilterBuilder implements 
FilterParser.FilterBuilder {
     }
 
     @Override
+    public Object build(FilterParser.Operation operation, Object operand) 
throws Exception {
+        return handleSimpleOperations(operation, (FilterParser.ColumnIndex) 
operand);
+    }
+
+    @Override
     public Object build(FilterParser.LogicalOperation opId, Object 
leftOperand, Object rightOperand) {
         return handleCompoundOperations(opId, (Filter) leftOperand, (Filter) 
rightOperand);
     }
@@ -167,6 +172,32 @@ public class HBaseFilterBuilder implements 
FilterParser.FilterBuilder {
     }
 
     /**
+     * Handles simple column-operator expressions.
+     */
+    private Filter handleSimpleOperations(FilterParser.Operation opId,
+                                          FilterParser.ColumnIndex column) 
throws Exception {
+        HBaseColumnDescriptor hbaseColumn = 
tupleDescription.getColumn(column.index());
+        CompareFilter.CompareOp compareOperation;
+        ByteArrayComparable comparator;
+        switch (opId) {
+            case HDOP_ISNULL:
+                compareOperation = CompareFilter.CompareOp.EQUAL;
+                comparator = new NullComparator();
+                break;
+            case HDOP_ISNOTNULL:
+                compareOperation = CompareFilter.CompareOp.NOT_EQUAL;
+                comparator = new NullComparator();
+                break;
+            default:
+                throw new Exception("unsupported unary operation for filtering 
" + opId);
+        }
+        return new SingleColumnValueFilter(hbaseColumn.columnFamilyBytes(),
+                hbaseColumn.qualifierBytes(),
+                compareOperation,
+                comparator);
+    }
+
+    /**
      * Handles simple column-operator-constant expressions.
      * Creates a special filter in the case the column is the row key column.
      */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
----------------------------------------------------------------------
diff --git 
a/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
 
b/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
index 9bc5474..0d9de3d 100644
--- 
a/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
+++ 
b/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
@@ -1,8 +1,15 @@
 package org.apache.hawq.pxf.plugins.hbase;
 
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.NullComparator;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hawq.pxf.plugins.hbase.utilities.HBaseColumnDescriptor;
+import org.apache.hawq.pxf.plugins.hbase.utilities.HBaseTupleDescription;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class HBaseFilterBuilderTest {
 
@@ -13,4 +20,52 @@ public class HBaseFilterBuilderTest {
         assertNull(builder.getFilterObject(filter));
     }
 
+    @Test
+    public void parseISNULLExpression() throws Exception {
+        String filter = "a1o9";
+        HBaseTupleDescription desc = mock(HBaseTupleDescription.class);
+        HBaseColumnDescriptor column = mock(HBaseColumnDescriptor.class);
+        when(desc.getColumn(1)).thenReturn(column);
+
+        byte[] family = new byte[]{};
+        byte[] qualifier = new byte[]{};
+
+        when(column.columnFamilyBytes()).thenReturn(family);
+        when(column.qualifierBytes()).thenReturn(qualifier);
+
+        HBaseFilterBuilder builder = new HBaseFilterBuilder(desc);
+        SingleColumnValueFilter result = (SingleColumnValueFilter) 
builder.getFilterObject(filter);
+
+        assertNotNull(result);
+        assertSame(family, result.getFamily());
+        assertSame(qualifier, result.getQualifier());
+        assertEquals(CompareFilter.CompareOp.EQUAL, result.getOperator());
+        assertTrue(result.getComparator() instanceof NullComparator);
+
+    }
+
+    @Test
+    public void parseISNOTNULLExpression() throws Exception {
+        String filter = "a1o10";
+        HBaseTupleDescription desc = mock(HBaseTupleDescription.class);
+        HBaseColumnDescriptor column = mock(HBaseColumnDescriptor.class);
+        when(desc.getColumn(1)).thenReturn(column);
+
+        byte[] family = new byte[]{};
+        byte[] qualifier = new byte[]{};
+
+        when(column.columnFamilyBytes()).thenReturn(family);
+        when(column.qualifierBytes()).thenReturn(qualifier);
+
+        HBaseFilterBuilder builder = new HBaseFilterBuilder(desc);
+        SingleColumnValueFilter result = (SingleColumnValueFilter) 
builder.getFilterObject(filter);
+
+        assertNotNull(result);
+        assertSame(family, result.getFamily());
+        assertSame(qualifier, result.getQualifier());
+        assertEquals(CompareFilter.CompareOp.NOT_EQUAL, result.getOperator());
+        assertTrue(result.getComparator() instanceof NullComparator);
+
+    }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java
----------------------------------------------------------------------
diff --git 
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java
 
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java
index bd82a3b..6d76036 100644
--- 
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java
+++ 
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilder.java
@@ -93,6 +93,18 @@ public class HiveFilterBuilder implements 
FilterParser.FilterBuilder {
                 (FilterParser.Constant) rightOperand);
     }
 
+    @Override
+    public Object build(FilterParser.Operation operation, Object operand) 
throws Exception {
+        if (operation == FilterParser.Operation.HDOP_ISNULL || operation == 
FilterParser.Operation.HDOP_ISNOTNULL) {
+            // use null for the constant value of null comparison
+            return handleSimpleOperations(operation,
+                    (FilterParser.ColumnIndex) operand,
+                    null);
+        } else {
+            throw new Exception("Unsupported unary operation " + operation);
+        }
+    }
+
     /*
      * Handles simple column-operator-constant expressions Creates a special
      * filter in the case the column is the row key column

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
----------------------------------------------------------------------
diff --git 
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
 
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
index ab2f96e..72ab4e1 100644
--- 
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
+++ 
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
@@ -143,7 +143,8 @@ public class HiveORCAccessor extends HiveAccessor {
         /* The below functions will not be compatible and requires update  
with Hive 2.0 APIs */
         BasicFilter filter = (BasicFilter) filterObj;
         int filterColumnIndex = filter.getColumn().index();
-        Object filterValue = filter.getConstant().constant();
+        // filter value might be null for unary operations
+        Object filterValue = filter.getConstant() == null ? null : 
filter.getConstant().constant();
         ColumnDescriptor filterColumn = inputData.getColumn(filterColumnIndex);
         String filterColumnName = filterColumn.columnName();
 
@@ -166,6 +167,12 @@ public class HiveORCAccessor extends HiveAccessor {
             case HDOP_NE:
                 builder.startNot().equals(filterColumnName, filterValue).end();
                 break;
+            case HDOP_ISNULL:
+                builder.isNull(filterColumnName);
+                break;
+            case HDOP_ISNOTNULL:
+                builder.startNot().isNull(filterColumnName).end();
+                break;
         }
         return;
     }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3f7894df/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java
----------------------------------------------------------------------
diff --git 
a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java
 
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java
index e0e6536..7954cd0 100755
--- 
a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java
+++ 
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveFilterBuilderTest.java
@@ -29,6 +29,7 @@ import org.apache.hawq.pxf.api.BasicFilter;
 import static org.apache.hawq.pxf.api.FilterParser.Operation;
 import static org.apache.hawq.pxf.api.FilterParser.Operation.*;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 public class HiveFilterBuilderTest {
     @Test
@@ -63,4 +64,22 @@ public class HiveFilterBuilderTest {
         assertEquals(HDOP_LT, ((BasicFilter) 
filter.getFilterList().get(1)).getOperation());
     }
 
+    @Test
+    public void parseISNULLExpression() throws Exception {
+        HiveFilterBuilder builder = new HiveFilterBuilder(null);
+        BasicFilter filter = (BasicFilter) builder.getFilterObject("a1o9");
+        assertEquals(Operation.HDOP_ISNULL, filter.getOperation());
+        assertEquals(1, filter.getColumn().index());
+        assertNull(filter.getConstant());
+    }
+
+    @Test
+    public void parseISNOTNULLExpression() throws Exception {
+        HiveFilterBuilder builder = new HiveFilterBuilder(null);
+        BasicFilter filter = (BasicFilter) builder.getFilterObject("a1o10");
+        assertEquals(Operation.HDOP_ISNOTNULL, filter.getOperation());
+        assertEquals(1, filter.getColumn().index());
+        assertNull(filter.getConstant());
+    }
+
 }

Reply via email to