Repository: incubator-hawq Updated Branches: refs/heads/master cd81289bc -> 6416b536d
HAWQ-779 support pxf filter pushdwon at the 'CREATE PLAN' stage ,and more filter type & op Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6416b536 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6416b536 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6416b536 Branch: refs/heads/master Commit: 6416b536d273b5add89482d50ea403c70572da80 Parents: cd81289 Author: Devin Jia <[email protected]> Authored: Fri Aug 5 15:05:50 2016 +0800 Committer: Goden Yao <[email protected]> Committed: Wed Sep 14 12:30:25 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hawq/pxf/api/FilterParser.java | 4 +++- .../org/apache/hawq/pxf/api/FilterParserTest.java | 8 ++++++++ .../hawq/pxf/plugins/hbase/HBaseFilterBuilder.java | 8 ++++++++ .../hawq/pxf/plugins/hive/HiveDataFragmenter.java | 12 ++++++++++++ src/backend/access/external/pxffilters.c | 15 +++++++++++++-- src/backend/access/external/test/pxffilters_test.c | 14 +++++++++++++- src/backend/optimizer/plan/createplan.c | 8 +++++++- src/include/access/pxffilters.h | 3 ++- 8 files changed, 66 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java index ae96d22..00fbf2b 100644 --- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java @@ -61,7 +61,8 @@ public class FilterParser { HDOP_GE, HDOP_EQ, HDOP_NE, - HDOP_AND + HDOP_AND, + HDOP_LIKE } /** @@ -392,6 +393,7 @@ public class FilterParser { operatorTranslationMap.put(5, Operation.HDOP_EQ); operatorTranslationMap.put(6, Operation.HDOP_NE); operatorTranslationMap.put(7, Operation.HDOP_AND); + operatorTranslationMap.put(8, Operation.HDOP_LIKE); return operatorTranslationMap; } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java index 0308f7f..1ded4a3 100644 --- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java +++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java @@ -215,6 +215,10 @@ public class FilterParserTest { filter = "a1c2o7"; op = Operation.HDOP_AND; runParseOneOperation("this filter was build from HDOP_AND", filter, op); + + filter = "a1c2o8"; + op = Operation.HDOP_LIKE; + runParseOneOperation("this filter was built from HDOP_LIKE", filter, op); } @Test @@ -247,6 +251,10 @@ public class FilterParserTest { filter = "c2a1o7"; op = Operation.HDOP_AND; runParseOneOperation("this filter was build from HDOP_AND using reverse!", filter, op); + + filter = "c2a1o8"; + op = Operation.HDOP_LIKE; + runParseOneOperation("this filter was build from HDOP_LIKE using reverse!", filter, op); } @Test http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java index 6dc271e..8eadc88 100644 --- a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java +++ b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java @@ -165,6 +165,14 @@ public class HBaseFilterBuilder implements FilterParser.FilterBuilder { ByteArrayComparable comparator = getComparator(hbaseColumn.columnTypeCode(), constant.constant()); + if(operatorsMap.get(opId) == null){ + //HBase does not support HDOP_LIKE, use 'NOT NULL' comparator + return new SingleColumnValueFilter(hbaseColumn.columnFamilyBytes(), + hbaseColumn.qualifierBytes(), + CompareFilter.CompareOp.NOT_EQUAL, + new NullComparator()); + } + /** * If row key is of type TEXT, allow filter in start/stop row key API in * HBaseAccessor/Scan object. http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java index ded9627..2fe31cd 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java @@ -25,6 +25,8 @@ import java.util.ListIterator; import java.util.Properties; import java.util.Set; import java.util.TreeSet; +import java.util.Map; +import java.util.HashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -37,6 +39,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; @@ -48,6 +51,7 @@ import org.apache.hawq.pxf.api.Fragment; import org.apache.hawq.pxf.api.Fragmenter; import org.apache.hawq.pxf.api.FragmentsStats; import org.apache.hawq.pxf.api.Metadata; +import org.apache.hawq.pxf.api.io.DataType; import org.apache.hawq.pxf.api.utilities.ColumnDescriptor; import org.apache.hawq.pxf.api.utilities.InputData; import org.apache.hawq.pxf.plugins.hdfs.utilities.HdfsUtilities; @@ -87,6 +91,7 @@ public class HiveDataFragmenter extends Fragmenter { // partition filtering private Set<String> setPartitions = new TreeSet<String>( String.CASE_INSENSITIVE_ORDER); + private Map<String, String> partitionkeyTypes = new HashMap<>(); /** * A Hive table unit - means a subset of the HIVE table, where we can say @@ -191,6 +196,7 @@ public class HiveDataFragmenter extends Fragmenter { // Save all hive partition names in a set for later filter match for (FieldSchema fs : tbl.getPartitionKeys()) { setPartitions.add(fs.getName()); + partitionkeyTypes.put(fs.getName(), fs.getType()); } LOG.debug("setPartitions :" + setPartitions); @@ -451,6 +457,12 @@ public class HiveDataFragmenter extends Fragmenter { return false; } + if (!partitionkeyTypes.get(filterColumnName).equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) { + LOG.debug("Filter type is not string type , ignore this filter for hive: " + + filter); + return false; + } + filtersString.append(prefix); filtersString.append(filterColumnName); filtersString.append(HIVE_API_EQ); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/access/external/pxffilters.c ---------------------------------------------------------------------- diff --git a/src/backend/access/external/pxffilters.c b/src/backend/access/external/pxffilters.c index 405e6d7..cdb2cb6 100644 --- a/src/backend/access/external/pxffilters.c +++ b/src/backend/access/external/pxffilters.c @@ -79,6 +79,7 @@ dbop_pxfop_map pxf_supported_opr[] = {665 /* text_le */, PXFOP_LE}, {667 /* text_ge */, PXFOP_GE}, {531 /* textlt */, PXFOP_NE}, + {1209 /* textlike */, PXFOP_LIKE}, /* int2 to int4 */ {Int24EqualOperator /* int24eq */, PXFOP_EQ}, @@ -126,7 +127,15 @@ dbop_pxfop_map pxf_supported_opr[] = {1871 /* int82gt */, PXFOP_GT}, {1872 /* int82le */, PXFOP_LE}, {1873 /* int82ge */, PXFOP_GE}, - {1869 /* int82ne */, PXFOP_NE} + {1869 /* int82ne */, PXFOP_NE}, + + /* date */ + {DateEqualOperator /* eq */, PXFOP_EQ}, + {1095 /* date_lt */, PXFOP_LT}, + {1097 /* date_gt */, PXFOP_GT}, + {1096 /* date_le */, PXFOP_LE}, + {1098 /* date_ge */, PXFOP_GE}, + {1094 /* date_ne */, PXFOP_NE} }; @@ -143,7 +152,8 @@ Oid pxf_supported_types[] = BPCHAROID, CHAROID, BYTEAOID, - BOOLOID + BOOLOID, + DATEOID }; /* @@ -559,6 +569,7 @@ const_to_str(Const *constval, StringInfo buf) case BPCHAROID: case CHAROID: case BYTEAOID: + case DATEOID: appendStringInfo(buf, "\\\"%s\\\"", extval); break; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/access/external/test/pxffilters_test.c ---------------------------------------------------------------------- diff --git a/src/backend/access/external/test/pxffilters_test.c b/src/backend/access/external/test/pxffilters_test.c index a5b3fb0..65b1aef 100644 --- a/src/backend/access/external/test/pxffilters_test.c +++ b/src/backend/access/external/test/pxffilters_test.c @@ -42,6 +42,7 @@ test__supported_filter_type(void **state) CHAROID, BYTEAOID, BOOLOID, + DATEOID, CIRCLEOID /* unsupported type */ }; @@ -61,7 +62,7 @@ test__supported_filter_type(void **state) /* go over pxf_supported_types array */ int nargs = sizeof(pxf_supported_types) / sizeof(Oid); - assert_int_equal(nargs, 12); + assert_int_equal(nargs, 13); for (i = 0; i < nargs; ++i) { assert_true(supported_filter_type(pxf_supported_types[i])); @@ -184,6 +185,7 @@ test__const_to_str__text(void **state) verify__const_to_str(false, "isn't", BPCHAROID, "\\\"isn't\\\""); verify__const_to_str(false, "funny", CHAROID, "\\\"funny\\\""); verify__const_to_str(false, "anymore", BYTEAOID, "\\\"anymore\\\""); + verify__const_to_str(false, "iamdate", DATEOID, "\\\"iamdate\\\""); } void @@ -523,6 +525,10 @@ test__pxf_serialize_filter_list__manyFilters(void **state) PXF_ATTR_CODE, 5, NULL, PXF_CONST_CODE, 0, "\"Winston\"", PXFOP_GE); + PxfFilterDesc* filter5 = build_filter( + PXF_ATTR_CODE, 6, NULL, + PXF_CONST_CODE, 0, "\"Eric-%\"", + PXFOP_LIKE); filter_list = lappend(filter_list, filter1); filter_list = lappend(filter_list, filter2); @@ -543,6 +549,12 @@ test__pxf_serialize_filter_list__manyFilters(void **state) assert_string_equal(result, "a1c1983o2a2c1985o1o7a3c\"George Orwell\"o5o7a4c\"Winston\"o4o7"); pfree(result); + filter_list = lappend(filter_list, filter5); + + result = pxf_serialize_filter_list(filter_list); + assert_string_equal(result, "a1c1983o2a2c1985o1o7a3c\"George Orwell\"o5o7a4c\"Winston\"o4o7a5c\"Eric-%\"o8o7"); + pfree(result); + pxf_free_filter_list(filter_list); filter_list = NIL; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/optimizer/plan/createplan.c ---------------------------------------------------------------------- diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index b5a5cea..d7efa62 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -1144,9 +1144,15 @@ static char** create_pxf_plan(char **segdb_file_map, RelOptInfo *rel, int total_ Relation relation = RelationIdGetRelation(planner_rt_fetch(scan_relid, ctx->root)->relid); - segdb_work_map = map_hddata_2gp_segments(uri_str, + if (pxf_enable_filter_pushdown){ + segdb_work_map = map_hddata_2gp_segments(uri_str, + total_segs, segs_participating, + relation, ctx->root->parse->jointree->quals); + }else{ + segdb_work_map = map_hddata_2gp_segments(uri_str, total_segs, segs_participating, relation, NULL); + } Assert(segdb_work_map != NULL); RelationClose(relation); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/include/access/pxffilters.h ---------------------------------------------------------------------- diff --git a/src/include/access/pxffilters.h b/src/include/access/pxffilters.h index 1409e39..f54c47c 100644 --- a/src/include/access/pxffilters.h +++ b/src/include/access/pxffilters.h @@ -44,7 +44,8 @@ typedef enum PxfOperatorCode PXFOP_GE, PXFOP_EQ, PXFOP_NE, - PXFOP_AND + PXFOP_AND, + PXFOP_LIKE } PxfOperatorCode;
