Author: hashutosh Date: Sat Mar 22 06:04:42 2014 New Revision: 1580177 URL: http://svn.apache.org/r1580177 Log: HIVE-6650 : hive.optimize.index.filter breaks non-index where with HBaseStorageHandler (Nick Dimiduk via Ashutosh Chauhan)
Modified: hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out Modified: hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=1580177&r1=1580176&r2=1580177&view=diff ============================================================================== --- hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (original) +++ hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java Sat Mar 22 06:04:42 2014 @@ -253,6 +253,8 @@ public class HiveHBaseTableInputFormat e private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary) throws IOException { + // TODO: assert iKey is HBaseSerDe#HBASE_KEY_COL + Scan scan = new Scan(); String filterExprSerialized = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR); if (filterExprSerialized == null) { @@ -270,18 +272,15 @@ public class HiveHBaseTableInputFormat e ExprNodeDesc residualPredicate = analyzer.analyzePredicate(filterExpr, searchConditions); - // There should be no residual since we already negotiated - // that earlier in HBaseStorageHandler.decomposePredicate. + // There should be no residual since we already negotiated that earlier in + // HBaseStorageHandler.decomposePredicate. However, with hive.optimize.index.filter + // OpProcFactory#pushFilterToStorageHandler pushes the original filter back down again. + // Since pushed-down filters are not ommitted at the higher levels (and thus the + // contract of negotiation is ignored anyway), just ignore the residuals. + // Re-assess this when negotiation is honored and the duplicate evaluation is removed. + // THIS IGNORES RESIDUAL PARSING FROM HBaseStorageHandler#decomposePredicate if (residualPredicate != null) { - throw new RuntimeException( - "Unexpected residual predicate " + residualPredicate.getExprString()); - } - - // There should be exactly one predicate since we already - // negotiated that also. - if (searchConditions.size() < 1 || searchConditions.size() > 2) { - throw new RuntimeException( - "Either one or two search conditions expected in push down"); + LOG.debug("Ignoring residual predicate " + residualPredicate.getExprString()); } // Convert the search condition into a restriction on the HBase scan @@ -292,7 +291,7 @@ public class HiveHBaseTableInputFormat e PrimitiveObjectInspector objInspector; Object writable; - try{ + try { objInspector = (PrimitiveObjectInspector)eval.initialize(null); writable = eval.evaluate(null); } catch (ClassCastException cce) { Modified: hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q?rev=1580177&r1=1580176&r2=1580177&view=diff ============================================================================== --- hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q (original) +++ hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q Sat Mar 22 06:04:42 2014 @@ -17,6 +17,12 @@ explain select * from hbase_pushdown whe select * from hbase_pushdown where key=90 and value like '%90%'; +set hive.optimize.index.filter=true; +-- with partial pushdown with optimization (HIVE-6650) +explain select * from hbase_pushdown where key=90 and value like '%90%'; +select * from hbase_pushdown where key=90 and value like '%90%'; +set hive.optimize.index.filter=false; + -- with two residuals explain select * from hbase_pushdown Modified: hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out?rev=1580177&r1=1580176&r2=1580177&view=diff ============================================================================== --- hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out (original) +++ hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out Sat Mar 22 06:04:42 2014 @@ -115,6 +115,52 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@hbase_pushdown #### A masked pattern was here #### 90 val_90 +PREHOOK: query: -- with partial pushdown with optimization (HIVE-6650) +explain select * from hbase_pushdown where key=90 and value like '%90%' +PREHOOK: type: QUERY +POSTHOOK: query: -- with partial pushdown with optimization (HIVE-6650) +explain select * from hbase_pushdown where key=90 and value like '%90%' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: hbase_pushdown + filterExpr: (key = 90) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: (value like '%90%') (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + +PREHOOK: query: select * from hbase_pushdown where key=90 and value like '%90%' +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_pushdown +#### A masked pattern was here #### +POSTHOOK: query: select * from hbase_pushdown where key=90 and value like '%90%' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_pushdown +#### A masked pattern was here #### +90 val_90 PREHOOK: query: -- with two residuals explain select * from hbase_pushdown