Repository: incubator-hawq Updated Branches: refs/heads/master 838cf50d3 -> c9072d2ad
HAWQ-1116. HAWQ/PXF bridge support for PPD for operators isNull and isNotNull Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c9072d2a Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c9072d2a Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c9072d2a Branch: refs/heads/master Commit: c9072d2ad78421b9b89d9df643f046b4a3dd3c78 Parents: 838cf50 Author: Shivram Mani <[email protected]> Authored: Wed Oct 26 10:13:18 2016 -0700 Committer: Shivram Mani <[email protected]> Committed: Wed Oct 26 10:13:18 2016 -0700 ---------------------------------------------------------------------- .../hawq/pxf/plugins/hive/HiveAccessor.java | 2 +- src/backend/access/external/pxffilters.c | 28 ++++++- .../access/external/test/pxffilters_test.c | 86 ++++++++++++++++++-- src/include/access/pxffilters.h | 4 +- 4 files changed, 109 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c9072d2a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java index b757371..ef9f76e 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java @@ -361,7 +361,7 @@ public class HiveAccessor extends HdfsSplittableDataAccessor { BasicFilter bFilter = (BasicFilter) filter; boolean isOperationEqual = (bFilter.getOperation() == FilterParser.Operation.HDOP_EQ); int columnIndex = bFilter.getColumn().index(); - String value = bFilter.getConstant().constant().toString(); + String value = bFilter.getConstant() == null ? null : bFilter.getConstant().constant().toString(); LOG.debug("isOperationEqual: " + isOperationEqual + " columnIndex: " + columnIndex + " value: " + value); } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c9072d2a/src/backend/access/external/pxffilters.c ---------------------------------------------------------------------- diff --git a/src/backend/access/external/pxffilters.c b/src/backend/access/external/pxffilters.c index f06b07f..3961b48 100644 --- a/src/backend/access/external/pxffilters.c +++ b/src/backend/access/external/pxffilters.c @@ -41,7 +41,7 @@ static List* append_attr_from_var(Var* var, List* attrs); static void enrich_trivial_expression(List *expressionItems); /* - * All supported HAWQ operators, and their respective HFDS operator code. + * All supported HAWQ operators, and their respective HDFS operator code. * Note that it is OK to use hardcoded OIDs, since these are all pinned * down system catalog operators. * see pg_operator.h @@ -249,6 +249,7 @@ pxf_make_expression_items_list(List *quals, Node *parent, int *logicalOpsNum) switch (tag) { case T_OpExpr: + case T_NullTest: { result = lappend(result, expressionItem); break; @@ -321,7 +322,7 @@ pxf_free_filter(PxfFilterDesc* filter) * as flattened tree. Operands and operators are represented with their * respective codes. Each filter is serialized as follows: * - * <attcode><attnum><constcode><constval><opercode><opernum> + * <attcode><attnum><constcode><constval><constsizecode><constsize><constdata><constvalue><opercode><opernum> * * Example filter list: * @@ -429,6 +430,29 @@ pxf_serialize_filter_list(List *expressionItems) appendStringInfo(resbuf, "%c%d", PXF_LOGICAL_OPERATOR_CODE, boolType); break; } + case T_NullTest: + { + elog(DEBUG1, "pxf_serialize_filter_list: node tag %d (T_NullTest)", tag); + NullTest *expr = (NullTest *) node; + + /* filter expression for T_NullTest will not have any constant value */ + if (expr->nulltesttype == IS_NULL) + { + appendStringInfo(resbuf, "%c%d%c%d", PXF_ATTR_CODE, ((Var *) expr->arg)->varattno - 1, PXF_OPERATOR_CODE, PXFOP_IS_NULL); + } + else if (expr->nulltesttype == IS_NOT_NULL) + { + appendStringInfo(resbuf, "%c%d%c%d", PXF_ATTR_CODE, ((Var *) expr->arg)->varattno - 1, PXF_OPERATOR_CODE, PXFOP_IS_NOTNULL); + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("internal error in pxffilters.c:pxf_serialize_" + "filter_list. Found a NullTest filter with incorrect NullTestType"))); + } + break; + } default: { elog(DEBUG5, "Skipping tag: %d", tag); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c9072d2a/src/backend/access/external/test/pxffilters_test.c ---------------------------------------------------------------------- diff --git a/src/backend/access/external/test/pxffilters_test.c b/src/backend/access/external/test/pxffilters_test.c index fa1db94..235a1d7 100644 --- a/src/backend/access/external/test/pxffilters_test.c +++ b/src/backend/access/external/test/pxffilters_test.c @@ -323,10 +323,32 @@ OpExpr* build_op_expr(void* left, void* right, int op) return expr; } -ExpressionItem* build_expression_item(int lattnum, Oid lattrtype, char* rconststr, Oid rattrtype, int op) { +NullTest* build_null_expr(Expr* arg, NullTestType nullType) +{ + NullTest *expr = (NullTest*) palloc0(sizeof(NullTest)); + expr->nulltesttype = nullType; + expr->arg = arg; + + expr->xpr.type = T_NullTest; + return expr; +} +ExpressionItem* build_null_expression_item(int attnum, Oid attrtype, NullTestType nullType) +{ ExpressionItem *expressionItem = (ExpressionItem*) palloc0(sizeof(ExpressionItem)); + Var *vararg = build_var(attrtype, attnum); + OpExpr *operationExpression = build_null_expr(vararg, nullType); + expressionItem->node = operationExpression; + expressionItem->processed = false; + expressionItem->parent = NULL; + + return expressionItem; +} + +ExpressionItem* build_expression_item(int lattnum, Oid lattrtype, char* rconststr, Oid rattrtype, int op) +{ + ExpressionItem *expressionItem = (ExpressionItem*) palloc0(sizeof(ExpressionItem)); Var *leftop = build_var(lattrtype, lattnum); Const *rightop = build_const(rattrtype, strdup(rconststr)); OpExpr *operationExpression = build_op_expr(leftop, rightop, op); @@ -387,7 +409,7 @@ test__opexpr_to_pxffilter__allSupportedTypes(void **state) /* NOTE: this test is not a use case - when the query includes * 'is null' or 'is not null' the qualifier code is T_NullTest and not T_OpExpr */ void -test__opexpr_to_pxffilter__attributeIsNull(void **state) +test__opexpr_to_pxffilter__attributeEqualsNull(void **state) { PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc)); Var *arg_var = build_var(INT2OID, 1); @@ -410,6 +432,17 @@ test__opexpr_to_pxffilter__attributeIsNull(void **state) pfree(expr); } +void +test__opexpr_to_pxffilter__attributeIsNull(void **state) +{ + PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc)); + Var *arg_var = build_var(INT2OID, 1); + NullTest *expr = build_null_expr(arg_var, IS_NULL); + + free(expr->arg); + pfree(expr); +} + /* * Test for a query with different types. * Types pairing are not checked, it is covered by the @@ -490,8 +523,8 @@ test__opexpr_to_pxffilter__unsupportedOpNot(void **state) pfree(expr); } -void test__pxf_serialize_filter_list__oneFilter(void **state) { - +void test__pxf_serialize_filter_list__oneFilter(void **state) +{ List* expressionItems = NIL; ExpressionItem* filterExpressionItem = build_expression_item(1, TEXTOID, "1984", TEXTOID, TextEqualOperator); @@ -507,6 +540,42 @@ void test__pxf_serialize_filter_list__oneFilter(void **state) { } +void test__pxf_serialize_fillter_list__nullFilter(void **state) +{ + + List* expressionItems = NIL; + + ExpressionItem* filterExpressionItem = build_null_expression_item(1, TEXTOID, IS_NULL); + + expressionItems = lappend(expressionItems, filterExpressionItem); + + char* result = pxf_serialize_filter_list(expressionItems); + assert_string_equal(result, "a0o8"); + + pxf_free_expression_items_list(expressionItems, true); + expressionItems = NIL; + pfree(result); + +} + +void test__pxf_serialize_fillter_list__nullFilter(void **state) +{ + + List* expressionItems = NIL; + + ExpressionItem* filterExpressionItem = build_null_expression_item(1, TEXTOID, IS_NULL); + + expressionItems = lappend(expressionItems, filterExpressionItem); + + char* result = pxf_serialize_filter_list(expressionItems); + assert_string_equal(result, "a0o8"); + + pxf_free_expression_items_list(expressionItems, true); + expressionItems = NIL; + pfree(result); + +} + void test__pxf_serialize_filter_list__manyFilters(void **state) { @@ -519,7 +588,7 @@ test__pxf_serialize_filter_list__manyFilters(void **state) ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID, "Eric-%", TEXTOID, 1209); ExpressionItem* expressionItem5 = build_expression_item(5, TEXTOID, "\"Ugly\" string with quotes", TEXTOID, TextEqualOperator); ExpressionItem* expressionItem6 = build_expression_item(6, TEXTOID, "", TEXTOID, TextEqualOperator); - + ExpressionItem* expressionItem7 = build_null_expression_item(7, TEXTOID, IS_NOT_NULL); expressionItems = lappend(expressionItems, expressionItem1); expressionItems = lappend(expressionItems, expressionItem2); @@ -527,9 +596,10 @@ test__pxf_serialize_filter_list__manyFilters(void **state) expressionItems = lappend(expressionItems, expressionItem4); expressionItems = lappend(expressionItems, expressionItem5); expressionItems = lappend(expressionItems, expressionItem6); + expressionItems = lappend(expressionItems, expressionItem7); result = pxf_serialize_filter_list(expressionItems); - assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with quoteso5a5c25s0do5"); + assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with quoteso5a5c25s0do5a4o9"); pfree(result); int trivialExpressionItems = expressionItems->length; @@ -557,12 +627,14 @@ main(int argc, char* argv[]) unit_test(test__opexpr_to_pxffilter__unary_expr), unit_test(test__opexpr_to_pxffilter__intGT), unit_test(test__opexpr_to_pxffilter__allSupportedTypes), - unit_test(test__opexpr_to_pxffilter__attributeIsNull), + unit_test(test__opexpr_to_pxffilter__attributeEqualsNull), unit_test(test__opexpr_to_pxffilter__differentTypes), unit_test(test__opexpr_to_pxffilter__unsupportedTypeCircle), unit_test(test__opexpr_to_pxffilter__twoVars), unit_test(test__opexpr_to_pxffilter__unsupportedOpNot), + unit_test(test__opexpr_to_pxffilter__attributeIsNull), unit_test(test__pxf_serialize_filter_list__oneFilter), + unit_test(test__pxf_serialize_fillter_list__nullFilter), unit_test(test__pxf_serialize_filter_list__manyFilters) }; return run_tests(tests); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c9072d2a/src/include/access/pxffilters.h ---------------------------------------------------------------------- diff --git a/src/include/access/pxffilters.h b/src/include/access/pxffilters.h index 00c77d3..f194966 100644 --- a/src/include/access/pxffilters.h +++ b/src/include/access/pxffilters.h @@ -44,7 +44,9 @@ typedef enum PxfOperatorCode PXFOP_GE, PXFOP_EQ, PXFOP_NE, - PXFOP_LIKE + PXFOP_LIKE, + PXFOP_IS_NULL, + PXFOP_IS_NOTNULL } PxfOperatorCode;
