Repository: incubator-trafodion Updated Branches: refs/heads/master 455008179 -> ff4b7dd73
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c5f243f/core/sql/regress/seabase/EXPECTED011 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/EXPECTED011 b/core/sql/regress/seabase/EXPECTED011 index 0835206..27de89a 100644 --- a/core/sql/regress/seabase/EXPECTED011 +++ b/core/sql/regress/seabase/EXPECTED011 @@ -7,7 +7,7 @@ >>invoke T011T1; -- Definition of Trafodion table TRAFODION.SCH.T011T1 --- Definition current Fri Oct 2 02:48:09 2015 +-- Definition current Wed Jan 6 23:31:01 2016 ( A INT NO DEFAULT NOT NULL NOT DROPPABLE @@ -142,7 +142,7 @@ A B ------------------------------------------------------------------ PLAN SUMMARY MODULE_NAME .............. DYNAMICALLY COMPILED STATEMENT_NAME ........... NOT NAMED -PLAN_ID .................. 212310514090586565 +PLAN_ID .................. 212318883062781505 ROWS_OUT ................. 7 EST_TOTAL_COST ........... 0.01 STATEMENT ................ select * from t011t1 where a >= 3 and a < 5; @@ -174,13 +174,13 @@ DESCRIPTION SEABASE_VOLATILE_TABLES ON HBASE_ASYNC_DROP_TABLE OFF HBASE_SERIALIZATION .... ON - HBASE_FILTER_PREDS ..... ON + HBASE_FILTER_PREDS ..... 2 TRAF_ALIGNED_ROW_FORMAT OFF TRAF_INDEX_CREATE_OPT ON SCHEMA ................. TRAFODION.SCH QUERY_CACHE ............ 0 GENERATE_EXPLAIN ....... ON - ObjectUIDs ............. 645219028244664647 + ObjectUIDs ............. 5568224739891142318 select_list ............ TRAFODION.SCH.T011T1.A, TRAFODION.SCH.T011T1.B @@ -203,6 +203,8 @@ DESCRIPTION cache_size ........... 100 probes ................. 1 rows_accessed ......... 11 + column_retrieved ....... #1:1,#1:2 + pushed_down_rpn ........ (#1:1>=?)(#1:1<?)AND key_columns ............ A --- SQL operation complete. @@ -1264,7 +1266,7 @@ METRIC_TEXT_TABLE >>invoke trafodion."_REPOS_".metric_query_table; -- Definition of Trafodion table TRAFODION."_REPOS_".METRIC_QUERY_TABLE --- Definition current Fri Oct 2 02:50:03 2015 +-- Definition current Wed Jan 6 23:33:14 2016 ( INSTANCE_ID INT UNSIGNED NO DEFAULT NOT NULL NOT @@ -1446,7 +1448,7 @@ METRIC_TEXT_TABLE >>-- get qid for the prepared stmt >>get qid for statement explstmt; -MXID11000001146212310514063700975000000000206U3333300_1922_EXPLSTMT +MXID11000009891212318883040903421000000000206U3333300_1923_EXPLSTMT --- SQL operation complete. >> @@ -1498,7 +1500,7 @@ SEQ_NUM OPERATOR --- SQL command prepared. >>get qid for statement explstmt2; -MXID11000001146212310514063700975000000000206U3333300_1933_EXPLSTMT2 +MXID11000009891212318883040903421000000000206U3333300_1934_EXPLSTMT2 --- SQL operation complete. >>set qid MXID123456 for explstmt2; @@ -2071,7 +2073,7 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD >>invoke t011t7; -- Definition of Trafodion table TRAFODION.SCH.T011T7 --- Definition current Fri Oct 2 02:50:45 2015 +-- Definition current Wed Jan 6 23:34:21 2016 ( A INT NO DEFAULT NOT NULL NOT DROPPABLE @@ -2088,7 +2090,7 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD >>invoke t011t7; -- Definition of Trafodion table TRAFODION.SCH.T011T7 --- Definition current Fri Oct 2 02:50:46 2015 +-- Definition current Wed Jan 6 23:34:25 2016 ( A INT NO DEFAULT NOT NULL NOT DROPPABLE @@ -2105,7 +2107,7 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD >>invoke t011t7; -- Definition of Trafodion table TRAFODION.SCH.T011T7 --- Definition current Fri Oct 2 02:50:47 2015 +-- Definition current Wed Jan 6 23:34:28 2016 ( A INT NO DEFAULT NOT NULL NOT DROPPABLE http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c5f243f/core/sql/regress/seabase/EXPECTED016 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/EXPECTED016 b/core/sql/regress/seabase/EXPECTED016 index 6c94b18..8547cff 100644 --- a/core/sql/regress/seabase/EXPECTED016 +++ b/core/sql/regress/seabase/EXPECTED016 @@ -179,9 +179,9 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD ------------------------------------------------------------------ PLAN SUMMARY MODULE_NAME .............. DYNAMICALLY COMPILED STATEMENT_NAME ........... ST1 -PLAN_ID .................. 212302388499931914 +PLAN_ID .................. 212318883773450737 ROWS_OUT ................. 1 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 STATEMENT ................ select f,count(f) from mcStatPart7 <<+ cardinality 10e8 >> group by f; @@ -192,7 +192,7 @@ ROOT ====================================== SEQ_NO 5 ONLY CHILD 4 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 0 @@ -214,7 +214,7 @@ DESCRIPTION SEABASE_VOLATILE_TABLES ON HBASE_ASYNC_DROP_TABLE OFF HBASE_SERIALIZATION .... ON - HBASE_FILTER_PREDS ..... ON + HBASE_FILTER_PREDS ..... 2 TRAF_ALIGNED_ROW_FORMAT OFF TRAF_INDEX_CREATE_OPT ON SCHEMA ................. TRAFODION.HBASE @@ -223,7 +223,7 @@ DESCRIPTION HBASE_MIN_BYTES_PER_ESP 1 USTAT_COLLECT_MC_SKEW_V ON PARALLEL_NUM_ESPS ...... 4 - ObjectUIDs ............. 119978596074759062 + ObjectUIDs ............. 1492467077764906944 select_list ............ TRAFODION.HBASE.MCSTATPART7.F, cast(sum(count_nonull(TRAFODION.HBASE.MCSTATPART7. F))) @@ -233,7 +233,7 @@ HASH_PARTIAL_GROUPBY_ROOT ================= SEQ_NO 4 ONLY CHILD 3 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION memory_quota ........... 1200 MB max_card_est ........... 1 @@ -248,7 +248,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -268,8 +268,8 @@ DESCRIPTION HASH_PARTIAL_GROUPBY_LEAF ================= SEQ_NO 2 ONLY CHILD 1 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 -EST_OPER_COST ............ 3.9 -EST_TOTAL_COST ......... 526.64 +EST_OPER_COST ............ 4.98 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -283,8 +283,8 @@ TRAFODION_SCAN ============================ SEQ_NO 1 NO CHILDREN TABLE_NAME ............... MCSTATPART7 REQUESTS_IN .............. 1 ROWS_OUT ..... 1,000,000,000 -EST_OPER_COST .......... 522.74 -EST_TOTAL_COST ......... 522.74 +EST_OPER_COST .......... 548.5 +EST_TOTAL_COST ......... 548.5 DESCRIPTION max_card_est ........... 1e+09 fragment_id ............ 2 @@ -296,6 +296,7 @@ DESCRIPTION cache_blocks ........... OFF probes ................. 1 rows_accessed .......... 1e+09 + column_retrieved ....... #1:2,#1:3 key_columns ............ _SALT_, A, B begin_key .............. (_SALT_ = (\:_sys_HostVarLoHashPart Hash2Distrib 8)), (A = <min>), (B = <min>) @@ -325,9 +326,9 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD ------------------------------------------------------------------ PLAN SUMMARY MODULE_NAME .............. DYNAMICALLY COMPILED STATEMENT_NAME ........... ST1 -PLAN_ID .................. 212302388500149365 +PLAN_ID .................. 212318883773834175 ROWS_OUT ................. 1 -EST_TOTAL_COST ......... 541.39 +EST_TOTAL_COST ......... 568.59 STATEMENT ................ select f,count(f) from mcStatPart5 <<+ cardinality 10e8 >> group by f; @@ -338,7 +339,7 @@ ROOT ====================================== SEQ_NO 5 ONLY CHILD 4 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0 -EST_TOTAL_COST ......... 541.39 +EST_TOTAL_COST ......... 568.59 DESCRIPTION max_card_est ........... 1 fragment_id ............ 0 @@ -360,7 +361,7 @@ DESCRIPTION SEABASE_VOLATILE_TABLES ON HBASE_ASYNC_DROP_TABLE OFF HBASE_SERIALIZATION .... ON - HBASE_FILTER_PREDS ..... ON + HBASE_FILTER_PREDS ..... 2 TRAF_ALIGNED_ROW_FORMAT OFF TRAF_INDEX_CREATE_OPT ON SCHEMA ................. TRAFODION.HBASE @@ -369,7 +370,7 @@ DESCRIPTION HBASE_MIN_BYTES_PER_ESP 1 USTAT_COLLECT_MC_SKEW_V ON PARALLEL_NUM_ESPS ...... 4 - ObjectUIDs ............. 119978596074761065 + ObjectUIDs ............. 1492467077764909533 select_list ............ TRAFODION.HBASE.MCSTATPART5.F, cast(sum(count_nonull(TRAFODION.HBASE.MCSTATPART5. F))) @@ -379,7 +380,7 @@ HASH_PARTIAL_GROUPBY_ROOT ================= SEQ_NO 4 ONLY CHILD 3 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 541.39 +EST_TOTAL_COST ......... 568.59 DESCRIPTION memory_quota ........... 1200 MB max_card_est ........... 1 @@ -394,7 +395,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 541.39 +EST_TOTAL_COST ......... 568.59 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -414,8 +415,8 @@ DESCRIPTION HASH_PARTIAL_GROUPBY_LEAF ================= SEQ_NO 2 ONLY CHILD 1 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 -EST_OPER_COST ............ 3.9 -EST_TOTAL_COST ......... 541.39 +EST_OPER_COST ............ 4.98 +EST_TOTAL_COST ......... 568.59 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -429,8 +430,8 @@ TRAFODION_SCAN ============================ SEQ_NO 1 NO CHILDREN TABLE_NAME ............... MCSTATPART5 REQUESTS_IN .............. 1 ROWS_OUT ..... 1,000,000,000 -EST_OPER_COST .......... 537.49 -EST_TOTAL_COST ......... 537.49 +EST_OPER_COST .......... 563.62 +EST_TOTAL_COST ......... 563.62 DESCRIPTION max_card_est ........... 1e+09 fragment_id ............ 2 @@ -442,6 +443,7 @@ DESCRIPTION cache_blocks ........... OFF probes ................. 1 rows_accessed .......... 1e+09 + column_retrieved ....... #1:2,#1:3 key_columns ............ _SALT_, A, B begin_key .............. (_SALT_ = (\:_sys_HostVarLoHashPart Hash2Distrib 8)), (A = <min>), (B = <min>) @@ -471,9 +473,9 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD ------------------------------------------------------------------ PLAN SUMMARY MODULE_NAME .............. DYNAMICALLY COMPILED STATEMENT_NAME ........... ST1 -PLAN_ID .................. 212302388500356018 +PLAN_ID .................. 212318883774174434 ROWS_OUT ................. 1 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 STATEMENT ................ select f,count(f) from mcStatPart3 <<+ cardinality 10e8 >> group by f; @@ -484,7 +486,7 @@ ROOT ====================================== SEQ_NO 5 ONLY CHILD 4 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 0 @@ -506,7 +508,7 @@ DESCRIPTION SEABASE_VOLATILE_TABLES ON HBASE_ASYNC_DROP_TABLE OFF HBASE_SERIALIZATION .... ON - HBASE_FILTER_PREDS ..... ON + HBASE_FILTER_PREDS ..... 2 TRAF_ALIGNED_ROW_FORMAT OFF TRAF_INDEX_CREATE_OPT ON SCHEMA ................. TRAFODION.HBASE @@ -515,7 +517,7 @@ DESCRIPTION HBASE_MIN_BYTES_PER_ESP 1 USTAT_COLLECT_MC_SKEW_V ON PARALLEL_NUM_ESPS ...... 4 - ObjectUIDs ............. 119978596074761721 + ObjectUIDs ............. 1492467077764910468 select_list ............ TRAFODION.HBASE.MCSTATPART3.F, cast(sum(count_nonull(TRAFODION.HBASE.MCSTATPART3. F))) @@ -525,7 +527,7 @@ HASH_PARTIAL_GROUPBY_ROOT ================= SEQ_NO 4 ONLY CHILD 3 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION memory_quota ........... 1200 MB max_card_est ........... 1 @@ -540,7 +542,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -560,8 +562,8 @@ DESCRIPTION HASH_PARTIAL_GROUPBY_LEAF ================= SEQ_NO 2 ONLY CHILD 1 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 -EST_OPER_COST ............ 3.9 -EST_TOTAL_COST ......... 526.64 +EST_OPER_COST ............ 4.98 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -575,8 +577,8 @@ TRAFODION_SCAN ============================ SEQ_NO 1 NO CHILDREN TABLE_NAME ............... MCSTATPART3 REQUESTS_IN .............. 1 ROWS_OUT ..... 1,000,000,000 -EST_OPER_COST .......... 522.74 -EST_TOTAL_COST ......... 522.74 +EST_OPER_COST .......... 548.5 +EST_TOTAL_COST ......... 548.5 DESCRIPTION max_card_est ........... 1e+09 fragment_id ............ 2 @@ -588,6 +590,7 @@ DESCRIPTION cache_blocks ........... OFF probes ................. 1 rows_accessed .......... 1e+09 + column_retrieved ....... #1:2,#1:3 key_columns ............ _SALT_, A, B begin_key .............. (_SALT_ = (\:_sys_HostVarLoHashPart Hash2Distrib 8)), (A = <min>), (B = <min>) @@ -617,9 +620,9 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD ------------------------------------------------------------------ PLAN SUMMARY MODULE_NAME .............. DYNAMICALLY COMPILED STATEMENT_NAME ........... ST1 -PLAN_ID .................. 212302388500563772 +PLAN_ID .................. 212318883774449563 ROWS_OUT ................. 1 -EST_TOTAL_COST ......... 497.84 +EST_TOTAL_COST ......... 523.92 STATEMENT ................ select f,count(f) from mcStatPart6 <<+ cardinality 10e8 >> group by f; @@ -630,7 +633,7 @@ ROOT ====================================== SEQ_NO 5 ONLY CHILD 4 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0 -EST_TOTAL_COST ......... 497.84 +EST_TOTAL_COST ......... 523.92 DESCRIPTION max_card_est ........... 1 fragment_id ............ 0 @@ -652,7 +655,7 @@ DESCRIPTION SEABASE_VOLATILE_TABLES ON HBASE_ASYNC_DROP_TABLE OFF HBASE_SERIALIZATION .... ON - HBASE_FILTER_PREDS ..... ON + HBASE_FILTER_PREDS ..... 2 TRAF_ALIGNED_ROW_FORMAT OFF TRAF_INDEX_CREATE_OPT ON SCHEMA ................. TRAFODION.HBASE @@ -661,7 +664,7 @@ DESCRIPTION HBASE_MIN_BYTES_PER_ESP 1 USTAT_COLLECT_MC_SKEW_V ON PARALLEL_NUM_ESPS ...... 4 - ObjectUIDs ............. 119978596074762449 + ObjectUIDs ............. 1492467077764911379 select_list ............ TRAFODION.HBASE.MCSTATPART6.F, cast(sum(count_nonull(TRAFODION.HBASE.MCSTATPART6. F))) @@ -671,7 +674,7 @@ HASH_PARTIAL_GROUPBY_ROOT ================= SEQ_NO 4 ONLY CHILD 3 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 497.84 +EST_TOTAL_COST ......... 523.92 DESCRIPTION memory_quota ........... 1200 MB max_card_est ........... 1 @@ -686,7 +689,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 497.84 +EST_TOTAL_COST ......... 523.92 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -706,8 +709,8 @@ DESCRIPTION HASH_PARTIAL_GROUPBY_LEAF ================= SEQ_NO 2 ONLY CHILD 1 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 -EST_OPER_COST ............ 3.9 -EST_TOTAL_COST ......... 497.84 +EST_OPER_COST ............ 4.98 +EST_TOTAL_COST ......... 523.92 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -721,8 +724,8 @@ TRAFODION_SCAN ============================ SEQ_NO 1 NO CHILDREN TABLE_NAME ............... MCSTATPART6 REQUESTS_IN .............. 1 ROWS_OUT ..... 1,000,000,000 -EST_OPER_COST .......... 493.94 -EST_TOTAL_COST ......... 493.94 +EST_OPER_COST .......... 518.94 +EST_TOTAL_COST ......... 518.94 DESCRIPTION max_card_est ........... 1e+09 fragment_id ............ 2 @@ -734,6 +737,7 @@ DESCRIPTION cache_blocks ........... OFF probes ................. 1 rows_accessed .......... 1e+09 + column_retrieved ....... #1:2,#1:3 key_columns ............ _SALT_, A, B begin_key .............. (_SALT_ = (\:_sys_HostVarLoHashPart Hash2Distrib 8)), (A = <min>), (B = <min>) @@ -763,9 +767,9 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD ------------------------------------------------------------------ PLAN SUMMARY MODULE_NAME .............. DYNAMICALLY COMPILED STATEMENT_NAME ........... ST1 -PLAN_ID .................. 212302388500780580 +PLAN_ID .................. 212318883774716355 ROWS_OUT ................. 1 -EST_TOTAL_COST ......... 497.84 +EST_TOTAL_COST ......... 523.92 STATEMENT ................ select f,count(f) from mcStatPart2 <<+ cardinality 10e8 >> group by f; @@ -776,7 +780,7 @@ ROOT ====================================== SEQ_NO 5 ONLY CHILD 4 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0 -EST_TOTAL_COST ......... 497.84 +EST_TOTAL_COST ......... 523.92 DESCRIPTION max_card_est ........... 1 fragment_id ............ 0 @@ -798,7 +802,7 @@ DESCRIPTION SEABASE_VOLATILE_TABLES ON HBASE_ASYNC_DROP_TABLE OFF HBASE_SERIALIZATION .... ON - HBASE_FILTER_PREDS ..... ON + HBASE_FILTER_PREDS ..... 2 TRAF_ALIGNED_ROW_FORMAT OFF TRAF_INDEX_CREATE_OPT ON SCHEMA ................. TRAFODION.HBASE @@ -807,7 +811,7 @@ DESCRIPTION HBASE_MIN_BYTES_PER_ESP 1 USTAT_COLLECT_MC_SKEW_V ON PARALLEL_NUM_ESPS ...... 4 - ObjectUIDs ............. 119978596074763069 + ObjectUIDs ............. 1492467077764912234 select_list ............ TRAFODION.HBASE.MCSTATPART2.F, cast(sum(count_nonull(TRAFODION.HBASE.MCSTATPART2. F))) @@ -817,7 +821,7 @@ HASH_PARTIAL_GROUPBY_ROOT ================= SEQ_NO 4 ONLY CHILD 3 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 497.84 +EST_TOTAL_COST ......... 523.92 DESCRIPTION memory_quota ........... 1200 MB max_card_est ........... 1 @@ -832,7 +836,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 497.84 +EST_TOTAL_COST ......... 523.92 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -852,8 +856,8 @@ DESCRIPTION HASH_PARTIAL_GROUPBY_LEAF ================= SEQ_NO 2 ONLY CHILD 1 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 -EST_OPER_COST ............ 3.9 -EST_TOTAL_COST ......... 497.84 +EST_OPER_COST ............ 4.98 +EST_TOTAL_COST ......... 523.92 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -867,8 +871,8 @@ TRAFODION_SCAN ============================ SEQ_NO 1 NO CHILDREN TABLE_NAME ............... MCSTATPART2 REQUESTS_IN .............. 1 ROWS_OUT ..... 1,000,000,000 -EST_OPER_COST .......... 493.94 -EST_TOTAL_COST ......... 493.94 +EST_OPER_COST .......... 518.94 +EST_TOTAL_COST ......... 518.94 DESCRIPTION max_card_est ........... 1e+09 fragment_id ............ 2 @@ -880,6 +884,7 @@ DESCRIPTION cache_blocks ........... OFF probes ................. 1 rows_accessed .......... 1e+09 + column_retrieved ....... #1:2,#1:3 key_columns ............ _SALT_, A, B begin_key .............. (_SALT_ = (\:_sys_HostVarLoHashPart Hash2Distrib 8)), (A = <min>), (B = <min>) @@ -909,9 +914,9 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD ------------------------------------------------------------------ PLAN SUMMARY MODULE_NAME .............. DYNAMICALLY COMPILED STATEMENT_NAME ........... ST1 -PLAN_ID .................. 212302388500996180 +PLAN_ID .................. 212318883775019433 ROWS_OUT ................. 1 -EST_TOTAL_COST ......... 511.84 +EST_TOTAL_COST ......... 538.3 STATEMENT ................ select f,count(f) from mcStatPart1 <<+ cardinality 10e8 >> group by f; @@ -922,7 +927,7 @@ ROOT ====================================== SEQ_NO 5 ONLY CHILD 4 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0 -EST_TOTAL_COST ......... 511.84 +EST_TOTAL_COST ......... 538.3 DESCRIPTION max_card_est ........... 1 fragment_id ............ 0 @@ -944,7 +949,7 @@ DESCRIPTION SEABASE_VOLATILE_TABLES ON HBASE_ASYNC_DROP_TABLE OFF HBASE_SERIALIZATION .... ON - HBASE_FILTER_PREDS ..... ON + HBASE_FILTER_PREDS ..... 2 TRAF_ALIGNED_ROW_FORMAT OFF TRAF_INDEX_CREATE_OPT ON SCHEMA ................. TRAFODION.HBASE @@ -953,7 +958,7 @@ DESCRIPTION HBASE_MIN_BYTES_PER_ESP 1 USTAT_COLLECT_MC_SKEW_V ON PARALLEL_NUM_ESPS ...... 4 - ObjectUIDs ............. 119978596074763689 + ObjectUIDs ............. 1492467077764913063 select_list ............ TRAFODION.HBASE.MCSTATPART1.F, cast(sum(count_nonull(TRAFODION.HBASE.MCSTATPART1. F))) @@ -963,7 +968,7 @@ HASH_PARTIAL_GROUPBY_ROOT ================= SEQ_NO 4 ONLY CHILD 3 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 511.84 +EST_TOTAL_COST ......... 538.3 DESCRIPTION memory_quota ........... 1200 MB max_card_est ........... 1 @@ -978,7 +983,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 511.84 +EST_TOTAL_COST ......... 538.3 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -998,8 +1003,8 @@ DESCRIPTION HASH_PARTIAL_GROUPBY_LEAF ================= SEQ_NO 2 ONLY CHILD 1 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 -EST_OPER_COST ............ 3.9 -EST_TOTAL_COST ......... 511.84 +EST_OPER_COST ............ 4.98 +EST_TOTAL_COST ......... 538.3 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -1013,8 +1018,8 @@ TRAFODION_SCAN ============================ SEQ_NO 1 NO CHILDREN TABLE_NAME ............... MCSTATPART1 REQUESTS_IN .............. 1 ROWS_OUT ..... 1,000,000,000 -EST_OPER_COST .......... 507.95 -EST_TOTAL_COST ......... 507.95 +EST_OPER_COST .......... 533.33 +EST_TOTAL_COST ......... 533.33 DESCRIPTION max_card_est ........... 1e+09 fragment_id ............ 2 @@ -1026,6 +1031,7 @@ DESCRIPTION cache_blocks ........... OFF probes ................. 1 rows_accessed .......... 1e+09 + column_retrieved ....... #1:2,#1:3 key_columns ............ _SALT_, A, B begin_key .............. (_SALT_ = (\:_sys_HostVarLoHashPart Hash2Distrib 8)), (A = <min>), (B = '<min>') @@ -1055,9 +1061,9 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD ------------------------------------------------------------------ PLAN SUMMARY MODULE_NAME .............. DYNAMICALLY COMPILED STATEMENT_NAME ........... ST1 -PLAN_ID .................. 212302388501209482 +PLAN_ID .................. 212318883775293423 ROWS_OUT ................. 1 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 STATEMENT ................ select f,count(f) from mcStatPart8 <<+ cardinality 10e8 >> group by f; @@ -1068,7 +1074,7 @@ ROOT ====================================== SEQ_NO 5 ONLY CHILD 4 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 0 @@ -1090,7 +1096,7 @@ DESCRIPTION SEABASE_VOLATILE_TABLES ON HBASE_ASYNC_DROP_TABLE OFF HBASE_SERIALIZATION .... ON - HBASE_FILTER_PREDS ..... ON + HBASE_FILTER_PREDS ..... 2 TRAF_ALIGNED_ROW_FORMAT OFF TRAF_INDEX_CREATE_OPT ON SCHEMA ................. TRAFODION.HBASE @@ -1099,7 +1105,7 @@ DESCRIPTION HBASE_MIN_BYTES_PER_ESP 1 USTAT_COLLECT_MC_SKEW_V ON PARALLEL_NUM_ESPS ...... 4 - ObjectUIDs ............. 119978596074764409 + ObjectUIDs ............. 1492467077764913972 select_list ............ TRAFODION.HBASE.MCSTATPART8.F, cast(sum(count_nonull(TRAFODION.HBASE.MCSTATPART8. F))) @@ -1109,7 +1115,7 @@ HASH_PARTIAL_GROUPBY_ROOT ================= SEQ_NO 4 ONLY CHILD 3 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION memory_quota ........... 1200 MB max_card_est ........... 1 @@ -1124,7 +1130,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 EST_OPER_COST ............ 0.01 -EST_TOTAL_COST ......... 526.64 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -1144,8 +1150,8 @@ DESCRIPTION HASH_PARTIAL_GROUPBY_LEAF ================= SEQ_NO 2 ONLY CHILD 1 REQUESTS_IN .............. 1 ROWS_OUT ................. 1 -EST_OPER_COST ............ 3.9 -EST_TOTAL_COST ......... 526.64 +EST_OPER_COST ............ 4.98 +EST_TOTAL_COST ......... 553.48 DESCRIPTION max_card_est ........... 1 fragment_id ............ 2 @@ -1159,8 +1165,8 @@ TRAFODION_SCAN ============================ SEQ_NO 1 NO CHILDREN TABLE_NAME ............... MCSTATPART8 REQUESTS_IN .............. 1 ROWS_OUT ..... 1,000,000,000 -EST_OPER_COST .......... 522.74 -EST_TOTAL_COST ......... 522.74 +EST_OPER_COST .......... 548.5 +EST_TOTAL_COST ......... 548.5 DESCRIPTION max_card_est ........... 1e+09 fragment_id ............ 2 @@ -1172,6 +1178,7 @@ DESCRIPTION cache_blocks ........... OFF probes ................. 1 rows_accessed .......... 1e+09 + column_retrieved ....... #1:2,#1:3 key_columns ............ _SALT_, A, B begin_key .............. (_SALT_ = (\:_sys_HostVarLoHashPart Hash2Distrib 8)), (A = <min>), (B = '<min>') http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c5f243f/core/sql/regress/tools/runregr_executor.ksh ---------------------------------------------------------------------- diff --git a/core/sql/regress/tools/runregr_executor.ksh b/core/sql/regress/tools/runregr_executor.ksh index a34af0f..5b77e35 100755 --- a/core/sql/regress/tools/runregr_executor.ksh +++ b/core/sql/regress/tools/runregr_executor.ksh @@ -237,7 +237,7 @@ skippedfiles= # sbtestfiles contains the list of tests to be run in seabase mode if [ "$seabase" -ne 0 ]; then - sbtestfiles="TEST001 TEST002 TEST012 TEST013 TEST014 TEST015 TEST016 TEST020 TEST022 TEST025 TEST050 TEST063 TEST088 TEST101 TEST106 TEST107 TEST122 TEST130" + sbtestfiles="TEST001 TEST002 TEST012 TEST013 TEST014 TEST015 TEST016 TEST020 TEST022 TEST025 TEST050 TEST063 TEST088 TEST101 TEST106 TEST107 TEST122 TEST130 TEST140" sbprettyfiles= for i in $prettyfiles; do for j in $sbtestfiles; do http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c5f243f/core/sql/regress/tools/sbdefs ---------------------------------------------------------------------- diff --git a/core/sql/regress/tools/sbdefs b/core/sql/regress/tools/sbdefs index 580d1e2..7fc062d 100644 --- a/core/sql/regress/tools/sbdefs +++ b/core/sql/regress/tools/sbdefs @@ -27,7 +27,7 @@ cqd seabase_volatile_tables 'ON'; cqd hbase_async_drop_table 'OFF'; cqd hbase_serialization 'ON'; --cqd hbase_coprocessors 'OFF'; -cqd hbase_filter_preds 'ON'; +cqd hbase_filter_preds '2'; cqd traf_aligned_row_format 'OFF'; --cqd traf_default_col_charset 'UTF8'; cqd traf_index_create_opt 'ON'; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c5f243f/core/sql/sqlcomp/DefaultConstants.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h index 1a988b4..e9f2427 100644 --- a/core/sql/sqlcomp/DefaultConstants.h +++ b/core/sql/sqlcomp/DefaultConstants.h @@ -3453,7 +3453,7 @@ enum DefaultConstants // instead of relying on default cardinality estimate. ESTIMATE_HBASE_ROW_COUNT, - // if ON, then filter predicates could be pushed down to hbase. + // if OFF or '0' is disabled, ON or '1' is simple pushdown, '2' is for advance pushdown // It will depends on the query on which predicates or sub-predicates could be pushed. HBASE_FILTER_PREDS, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c5f243f/core/sql/sqlcomp/nadefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp index 24b8681..9de9f4c 100644 --- a/core/sql/sqlcomp/nadefaults.cpp +++ b/core/sql/sqlcomp/nadefaults.cpp @@ -6428,6 +6428,14 @@ DefaultToken NADefaults::token(Int32 attrEnum, case '2': return DF_HIGH; case '3': return DF_MAXIMUM; } + // HBASE_FILTER_PREDS + if ((attrEnum == HBASE_FILTER_PREDS) && value.length()==1) + switch (*value.data()){ + case '0': return DF_OFF; + case '1': return DF_MINIMUM; + case '2': return DF_MEDIUM; + // in the future add DF_HIGH and DF_MAXIMUM when we implement more pushdown capabilities + } if ( attrEnum == TEMPORARY_TABLE_HASH_PARTITIONS || attrEnum == MVQR_REWRITE_CANDIDATES || attrEnum == MVQR_PUBLISH_TABLE_LOCATION || @@ -6684,6 +6692,15 @@ DefaultToken NADefaults::token(Int32 attrEnum, isValid = TRUE; break; + case HBASE_FILTER_PREDS: + if(tok == DF_OFF || tok == DF_ON) + { + if (tok == DF_ON) + tok = DF_MINIMUM; // to keep backward compatibility + isValid= TRUE; + } + break; + case ROBUST_QUERY_OPTIMIZATION: if (tok == DF_MINIMUM || tok == DF_SYSTEM || tok == DF_MAXIMUM || tok == DF_HIGH) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c5f243f/core/sql/src/main/java/org/trafodion/sql/HTableClient.java ---------------------------------------------------------------------- diff --git a/core/sql/src/main/java/org/trafodion/sql/HTableClient.java b/core/sql/src/main/java/org/trafodion/sql/HTableClient.java index 12d448a..5d936bc 100644 --- a/core/sql/src/main/java/org/trafodion/sql/HTableClient.java +++ b/core/sql/src/main/java/org/trafodion/sql/HTableClient.java @@ -73,12 +73,19 @@ import org.apache.hadoop.hbase.util.*; import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter; import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter; -// classes to do column value filtering +// classes to do hbase pushdown filtering import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.RandomRowFilter; +import org.apache.hadoop.hbase.filter.FamilyFilter; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.BinaryPrefixComparator; +import org.apache.hadoop.hbase.filter.ByteArrayComparable; +import org.apache.hadoop.hbase.filter.NullComparator; import org.apache.hadoop.hbase.client.TableSnapshotScanner; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -358,6 +365,417 @@ public class HTableClient { table.setAutoFlush(true, true); } + private enum Op { + EQUAL, EQUAL_NULL, NOT_EQUAL, NOT_EQUAL_NULL, LESS, LESS_NULL, LESS_OR_EQUAL, LESS_OR_EQUAL_NULL, GREATER, GREATER_NULL, + GREATER_OR_EQUAL, GREATER_OR_EQUAL_NULL, NO_OP, NO_OP_NULL,IS_NULL, IS_NULL_NULL, IS_NOT_NULL, IS_NOT_NULL_NULL, AND, OR}; + + private Filter SingleColumnValueExcludeOrNotFilter(byte[] columnToFilter, + CompareOp op, + ByteArrayComparable comparator, + HashMap<String,Object> columnsToRemove, + Boolean... filterIfMissing){ + Filter result; + boolean fMissing = filterIfMissing.length>0?filterIfMissing[0]:false;//default to false + if ((columnsToRemove == null) || !columnsToRemove.containsKey(new String(columnToFilter))){ + result = new SingleColumnValueFilter(getFamily(columnToFilter), getName(columnToFilter), op, comparator); + ((SingleColumnValueFilter)result).setFilterIfMissing(fMissing); + } + else{ + result= new SingleColumnValueExcludeFilter(getFamily(columnToFilter), getName(columnToFilter), op, comparator); + ((SingleColumnValueExcludeFilter)result).setFilterIfMissing(fMissing); + } + return result; + } + + // construct the hbase filter + // optimizes for OR and AND associativity + // optimizes for detection of a<? and a>? on nullable and non nullable column equivalent to a<>? + // optimize for null check factorization (A not null and (A <op> ?)) or (A not null and A <op2> ?) -> A not null and (A <op> ? or A <op2> ?) + // this is an important optimzation for IN statement on non null column + // uses the columnToRemove parametter to know if we need to use the SingleColumnValue Exclude or not method to limit returned columns + + private Filter constructV2Filter(Object[] colNamesToFilter, + Object[] compareOpList, + Object[] colValuesToCompare, + HashMap<String,Object> columnsToRemove){ + LinkedList linkedList = new LinkedList(); + //populate the list with nodes in reverse polish notation order. + int k=0;//column index + int kk=0;//value index + for (int i=1; i<compareOpList.length; i++){ // skip first one containing "V2" marker + String opStr = new String((byte[])compareOpList[i]); + switch(Op.valueOf(opStr)){ + + case EQUAL: + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove + )); + k++;kk++; + break; + case EQUAL_NULL: + linkedList.addLast(new FilterList(FilterList.Operator.MUST_PASS_ALL, //AND between if not null and the actual + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{0x00}),//check for null indicator = 0 representing non null + columnsToRemove, + true //filterIfMissing + ), + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove))); + k++;kk++; + break; + case NOT_EQUAL: + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.NOT_EQUAL, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove)); + k++;kk++; + break; + case NOT_EQUAL_NULL: + linkedList.addLast(new FilterList(FilterList.Operator.MUST_PASS_ALL, //AND between if not null and the actual + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{0x00}),//check for null indicator = 0 representing non null + columnsToRemove, + true), //filterIfMissing, + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.NOT_EQUAL, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove))); + k++;kk++; + break; + case LESS: + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.LESS, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove)); + k++;kk++; + break; + case LESS_NULL: + linkedList.addLast(new FilterList(FilterList.Operator.MUST_PASS_ALL, //AND between if not null and the actual + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{0x00}),//check for null indicator = 0 representing non null + columnsToRemove, + true), //filterIfMissing, + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.LESS, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove))); + k++;kk++; + break; + case LESS_OR_EQUAL: + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.LESS_OR_EQUAL, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove)); + k++;kk++; + break; + case LESS_OR_EQUAL_NULL: + linkedList.addLast(new FilterList(FilterList.Operator.MUST_PASS_ALL, //AND between if not null and the actual + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{0x00}),//check for null indicator = 0 representing non null + columnsToRemove, + true), //filterIfMissing, + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.LESS_OR_EQUAL, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove))); + k++;kk++; + break; + case GREATER: + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.GREATER, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove)); + k++;kk++; + break; + case GREATER_NULL: + linkedList.addLast(new FilterList(FilterList.Operator.MUST_PASS_ALL, //AND between if not null and the actual + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{0x00}),//check for null indicator = 0 representing non null + columnsToRemove, + true), //filterIfMissing, + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.GREATER, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove))); + k++;kk++; + break; + case GREATER_OR_EQUAL: + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.GREATER_OR_EQUAL, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove)); + k++;kk++; + break; + case GREATER_OR_EQUAL_NULL: + linkedList.addLast(new FilterList(FilterList.Operator.MUST_PASS_ALL, //AND between if not null and the actual + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{0x00}),//check for null indicator = 0 representing non null + columnsToRemove, + true), //filterIfMissing, + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.GREATER_OR_EQUAL, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove))); + k++;kk++; + break; + case NO_OP: + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.NO_OP, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove)); + k++;kk++; + break; + case NO_OP_NULL: + linkedList.addLast(new FilterList(FilterList.Operator.MUST_PASS_ALL, //AND between if not null and the actual + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{0x00}),//check for null indicator = 0 representing non null + columnsToRemove, + true), //filterIfMissing, + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.NO_OP, + new BinaryComparator((byte[])colValuesToCompare[kk]), + columnsToRemove))); + k++;kk++; + break; + case IS_NULL: + // is null on a non nullable column! + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.NO_OP, //exclude everything + new BinaryPrefixComparator((new byte[]{})), + columnsToRemove)); + k++; + break; + case IS_NULL_NULL: + // is_null on nullable column: is absent OR has the first byte set to FF indicating NULL. + linkedList.addLast( + new FilterList(FilterList.Operator.MUST_PASS_ONE, //OR + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new NullComparator(),//is absent? + columnsToRemove), + SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{-1}),//0xFF has null prefix indicator + columnsToRemove))); + k++; + break; + case IS_NOT_NULL: + // is not null on a non nullable column! + // do nothing, always true + k++; + break; + case IS_NOT_NULL_NULL: + // is_not_null on nullable column: is not absent AND has the first byte not set to FF indicating NULL. + linkedList.addLast(SingleColumnValueExcludeOrNotFilter( + (byte[])colNamesToFilter[k], + CompareOp.NOT_EQUAL, + new BinaryPrefixComparator(new byte[]{-1}),// 0xFF has null prefix indicator + columnsToRemove, + true));//filter if missing (if absent null) + k++; + break; + case AND: + linkedList.addLast("AND"); + break; + case OR: + linkedList.addLast("OR"); + break; + default: + }//switch + }//for + //evaluate the reverse polish notation list + while (linkedList.size()>1){// evaluate until only one element is left in the list + //look for first operator (AND or OR) + int j=0; + while (j<linkedList.size() && !(linkedList.get(j) instanceof String)){ + j++; + } + //here j points on the first operator; (all operands are of type Filter) + if (j==linkedList.size()){logger.error("j==linkedList.size()");return null;} // should not happen + Filter leftOperand; + Filter rightOperand; + switch(Op.valueOf((String)linkedList.get(j))){ + case AND: + FilterList filterListAnd = new FilterList(FilterList.Operator.MUST_PASS_ALL); //AND filterList + //left operand + leftOperand = (Filter)linkedList.get(j-2); + if (leftOperand instanceof FilterList && ((FilterList)leftOperand).getOperator()==FilterList.Operator.MUST_PASS_ALL){//associativity of AND optimization + //for(Filter f:((FilterList)leftOperand).getFilters()) + // filterListAnd.addFilter(f); + filterListAnd = (FilterList)leftOperand; //more efficient than the 2 lines above (kept commented out for code lisibility) + }else{ + filterListAnd.addFilter(leftOperand); + } + // right operand + rightOperand = (Filter)linkedList.get(j-1); + if (rightOperand instanceof FilterList && ((FilterList)rightOperand).getOperator()==FilterList.Operator.MUST_PASS_ALL){//associativity of AND optimization + for(Filter f:((FilterList)rightOperand).getFilters()) + filterListAnd.addFilter(f); + }else{ + filterListAnd.addFilter(rightOperand); + } + // setup evaluated filter + linkedList.set(j,filterListAnd); // replace the operator with the constructer filter + linkedList.remove(j-1);// remove right operand + linkedList.remove(j-2);// remove left operand. warning order matter + break; + case OR: + FilterList filterListOr = new FilterList(FilterList.Operator.MUST_PASS_ONE); //OR filterList + leftOperand = (Filter)linkedList.get(j-2); + rightOperand = (Filter)linkedList.get(j-1); + //begin detection of null check factorization (A not null and (A <op> ?)) or (A not null and A <op2> ?) -> A not null and (A <op> ? or A <op2> ?) + //the code is doing more than just nullcheck, but any factorization where left operands are identical + if (leftOperand instanceof FilterList && rightOperand instanceof FilterList && + ((FilterList)leftOperand).getOperator() == FilterList.Operator.MUST_PASS_ALL && + ((FilterList)rightOperand).getOperator() == FilterList.Operator.MUST_PASS_ALL && + ((FilterList)leftOperand).getFilters().size() == 2 && + ((FilterList)rightOperand).getFilters().size() == 2 && + ((FilterList)leftOperand).getFilters().get(0) instanceof SingleColumnValueFilter && //cannot be SingleColumnValueExcludeFilter when we have the optimization scenario + ((FilterList)rightOperand).getFilters().get(0) instanceof SingleColumnValueFilter){//cannot be SingleColumnValueExcludeFilter when we have the optimization scenario + SingleColumnValueFilter scvfLeft = (SingleColumnValueFilter)((FilterList)leftOperand).getFilters().get(0); + SingleColumnValueFilter scvfRight = (SingleColumnValueFilter)((FilterList)rightOperand).getFilters().get(0); + if (scvfLeft.getOperator() == scvfRight.getOperator() && //more general case than just for null check (identical operands) + Arrays.equals(scvfLeft.getQualifier(),scvfRight.getQualifier()) && + Arrays.equals(scvfLeft.getFamily(),scvfRight.getFamily()) && + Arrays.equals(scvfLeft.getComparator().getValue(),scvfRight.getComparator().getValue()) && + (scvfLeft.getFilterIfMissing() == scvfRight.getFilterIfMissing())){ + Filter left = ((FilterList)leftOperand).getFilters().get(1); + Filter right = ((FilterList)rightOperand).getFilters().get(1); + if (left instanceof FilterList && ((FilterList)left).getOperator()==FilterList.Operator.MUST_PASS_ONE){//associativity of OR optimization + //for(Filter f:((FilterList)left).getFilters()) + // filterListOr.addFilter(f); + filterListOr = (FilterList)left; // more efficient than the 2 lines above (kept commented out for code lisibility) + }else{ + filterListOr.addFilter(left); + } + // right operand + if (right instanceof FilterList && ((FilterList)right).getOperator()==FilterList.Operator.MUST_PASS_ONE){//associativity of OR optimization + for(Filter f:((FilterList)right).getFilters()) + filterListOr.addFilter(f); + }else{ + filterListOr.addFilter(right); + } + linkedList.set(j,new FilterList(FilterList.Operator.MUST_PASS_ALL,scvfLeft,filterListOr));//resulting factorized AND filter + linkedList.remove(j-1);// remove right operand + linkedList.remove(j-2);// remove left operand. warning order matter + break; + } + } + //end detection of null (and more) check factorization + //begin detection of RangeSpec a<>? transformed to a<? or a>? to convert it back to a <> ? when we push down + //check for <> on non nullable columns + if (leftOperand instanceof SingleColumnValueFilter && rightOperand instanceof SingleColumnValueFilter){ + SingleColumnValueFilter leftscvf = (SingleColumnValueFilter)leftOperand; + SingleColumnValueFilter rightscvf = (SingleColumnValueFilter)rightOperand; + if (leftscvf.getOperator() == CompareOp.LESS && rightscvf.getOperator()== CompareOp.GREATER && + Arrays.equals(leftscvf.getQualifier(), rightscvf.getQualifier()) && + Arrays.equals(leftscvf.getFamily(), rightscvf.getFamily()) && + Arrays.equals(leftscvf.getComparator().getValue(),rightscvf.getComparator().getValue()) + ){ + // setup evaluated filter + linkedList.set(j,new SingleColumnValueFilter(leftscvf.getFamily(), leftscvf.getQualifier(), CompareOp.NOT_EQUAL, leftscvf.getComparator())); // replace the operator with the constructer filter + linkedList.remove(j-1);// remove right operand + linkedList.remove(j-2);// remove left operand. warning order matter + break; + } + } + //check for <> on nullable column + if( leftOperand instanceof FilterList && rightOperand instanceof FilterList){ + //no need to check FilterList size, as all possible case FilterList size is at least 2. + if (((FilterList)leftOperand).getFilters().get(1) instanceof SingleColumnValueFilter && + ((FilterList)rightOperand).getFilters().get(1) instanceof SingleColumnValueFilter){ + SingleColumnValueFilter leftscvf = (SingleColumnValueFilter)((FilterList)leftOperand).getFilters().get(1); + SingleColumnValueFilter rightscvf = (SingleColumnValueFilter)((FilterList)rightOperand).getFilters().get(1); + if (leftscvf.getOperator() == CompareOp.LESS && rightscvf.getOperator()== CompareOp.GREATER && + Arrays.equals(leftscvf.getQualifier(), rightscvf.getQualifier()) && + Arrays.equals(leftscvf.getFamily(), rightscvf.getFamily()) && + Arrays.equals(leftscvf.getComparator().getValue(),rightscvf.getComparator().getValue()) + ){ + // setup evaluated filter + SingleColumnValueFilter nullCheck = new SingleColumnValueFilter(// null checker + leftscvf.getFamily(), leftscvf.getQualifier(), + CompareOp.EQUAL, + new BinaryPrefixComparator(new byte[]{0x00})); + nullCheck.setFilterIfMissing(true); + linkedList.set(j,new FilterList(FilterList.Operator.MUST_PASS_ALL, //AND between if not null and the actual + nullCheck, + new SingleColumnValueFilter( + leftscvf.getFamily(), leftscvf.getQualifier(), + CompareOp.NOT_EQUAL, + leftscvf.getComparator()))); + linkedList.remove(j-1);// remove right operand + linkedList.remove(j-2);// remove left operand. warning order matter + break; + } + } + } + //end detection of RangeSpec a<>? + //now general case... + //left operand + if (leftOperand instanceof FilterList && ((FilterList)leftOperand).getOperator()==FilterList.Operator.MUST_PASS_ONE){//associativity of OR optimization + //for(Filter f:((FilterList)leftOperand).getFilters()) + // filterListOr.addFilter(f); + filterListOr = (FilterList)leftOperand; // more efficient than the 2 lines above (kept commented out for code lisibility) + }else{ + filterListOr.addFilter(leftOperand); + } + // right operand + if (rightOperand instanceof FilterList && ((FilterList)rightOperand).getOperator()==FilterList.Operator.MUST_PASS_ONE){//associativity of OR optimization + for(Filter f:((FilterList)rightOperand).getFilters()) + filterListOr.addFilter(f); + }else{ + filterListOr.addFilter(rightOperand); + } + // setup evaluated filter + linkedList.set(j,filterListOr); // replace the operator with the constructer filter + linkedList.remove(j-1);// remove right operand + linkedList.remove(j-2);// remove left operand. warning order matter + break; + default: + logger.error("operator different than OR or AND???"); + return null;//should never happen + } + } + // after evaluation, the linkedList contains only one element containing the filter built + return (Filter)linkedList.pop(); + } + + public boolean startScan(long transID, byte[] startRow, byte[] stopRow, Object[] columns, long timestamp, boolean cacheBlocks, int numCacheRows, @@ -421,29 +839,140 @@ public class HTableClient { else numColsInScan = 0; if (colNamesToFilter != null) { - FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL); - - for (int i = 0; i < colNamesToFilter.length; i++) { - byte[] colName = (byte[])colNamesToFilter[i]; - byte[] coByte = (byte[])compareOpList[i]; - byte[] colVal = (byte[])colValuesToCompare[i]; - - if ((coByte == null) || (colVal == null)) { - return false; - } - - String coStr = new String(coByte); - CompareOp co = CompareOp.valueOf(coStr); - - SingleColumnValueFilter filter1 = - new SingleColumnValueFilter(getFamily(colName), getName(colName), - co, colVal); - list.addFilter(filter1); - } - + FilterList list; + boolean narrowDownResultColumns = false; //to check if we need a narrow down column filter (V2 only feature) + if (compareOpList == null)return false; + if (new String((byte[])compareOpList[0]).equals("V2")){ // are we dealing with predicate pushdown V2 + list = new FilterList(FilterList.Operator.MUST_PASS_ALL); + HashMap<String,Object> columnsToRemove = new HashMap<String,Object>(); + //if columnsToRemove not null, we are narrowing down using the SingleColumnValue[Exclude]Filter method + //else we will use the explicit FamilyFilter and QualifierFilter + //the simplified logic is that we can use the first method if and only if each and every column in the + //pushed down predicate shows up only once. + for (int i = 0; i < colNamesToFilter.length; i++) { + byte[] colName = (byte[])colNamesToFilter[i]; + + // check if the filter column is already part of the column list, if not add it if we are limiting columns (not *) + if(columns!=null && columns.length > 0){// if not * + boolean columnAlreadyIn = false; //assume column not yet in the scan object + for (int k=0; k<columns.length;k++){ + if (Arrays.equals(colName, (byte[])columns[k])){ + columnAlreadyIn = true;//found already exist + break;//no need to look further + } + } + if (!columnAlreadyIn){// column was not already in, so add it + scan.addColumn(getFamily(colName),getName(colName)); + narrowDownResultColumns = true; //since we added a column for predicate eval, we need to remove it later out of result set + String strColName = new String(colName); + if (columnsToRemove != null && columnsToRemove.containsKey(strColName)){// if we already added this column, it means it shows up more than once + columnsToRemove = null; // therefore, use the FamilyFilter/QualifierFilter method + }else if (columnsToRemove != null)// else + columnsToRemove.put(strColName,null); // add it to the list of column that should be nuked with the Exclude version of the SingleColumnValueFilter + } + } + } + if (columnsToRemove != null) + { //we are almost done checking if Exclude version of SingleColumnnValueFilter can be used. Th elast check s about to know if there is a IS_NULL_NULL + //operation that cannot be using the Exclude method, as it is transformed in a filterList with OR, therefore we cannot guaranty that the SingleColumnValueExcludeFilter + //performing the exclusion will be reached. + boolean is_null_nullFound = false; + for (Object o:compareOpList ){ + if (new String((byte[])o).equals("IS_NULL_NULL")){ + is_null_nullFound = true; + break; + } + } + if (is_null_nullFound){ + columnsToRemove = null; // disable Exclude method version of SingleColumnnValueFilter + }else + narrowDownResultColumns = false; // we will use the Exclude version of SingleColumnnValueFilter, so bypass the Family/QualifierFilter method + } + Filter f =constructV2Filter(colNamesToFilter,compareOpList,colValuesToCompare, columnsToRemove); + if (f==null) return false; // error logging done inside constructV2Filter + list.addFilter(f); + }//end V2 + else{// deal with V1 + list = new FilterList(FilterList.Operator.MUST_PASS_ALL); + + for (int i = 0; i < colNamesToFilter.length; i++) { + byte[] colName = (byte[])colNamesToFilter[i]; + byte[] coByte = (byte[])compareOpList[i]; + byte[] colVal = (byte[])colValuesToCompare[i]; + + if ((coByte == null) || (colVal == null)) { + return false; + } + String coStr = new String(coByte); + CompareOp co = CompareOp.valueOf(coStr); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter(getFamily(colName), getName(colName), + co, colVal); + list.addFilter(filter1); + } + }//end V1 + // if we added a column for predicate eval, we need to filter down result columns + FilterList resultColumnsOnlyFilter = null; + if (narrowDownResultColumns){ + HashMap<String,ArrayList<byte[]>> hm = new HashMap<String,ArrayList<byte[]>>(3);//use to deal with multiple family table + // initialize hm with list of columns requested for output + for (int i=0; i<columns.length; i++){ // if we are here we know columns is not null + if (hm.containsKey(new String(getFamily((byte[])columns[i])))){ + hm.get(new String(getFamily((byte[])columns[i]))).add((byte[])columns[i]); + }else{ + ArrayList<byte[]> al = new ArrayList<byte[]>(); + al.add((byte[])columns[i]); + hm.put(new String(getFamily((byte[])columns[i])), al); + } + } + + if (hm.size()==1){//only one column family + resultColumnsOnlyFilter = new FilterList(FilterList.Operator.MUST_PASS_ALL); + if (columns.length == 1){ + resultColumnsOnlyFilter.addFilter(new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(getName((byte[])columns[0])))); + }else{// more than one column + FilterList flColumns = new FilterList(FilterList.Operator.MUST_PASS_ONE); + for(int i=0; i<columns.length;i++) + flColumns.addFilter(new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(getName((byte[])columns[i])))); + resultColumnsOnlyFilter.addFilter(flColumns); + } + // note the optimization puting family check at the end + resultColumnsOnlyFilter.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(getFamily((byte[])columns[0])))); + }else{//more than one column family + resultColumnsOnlyFilter = new FilterList(FilterList.Operator.MUST_PASS_ONE); + for (Map.Entry<String,ArrayList<byte[]>> entry : hm.entrySet()){//for each column family + ArrayList<byte[]> alb = entry.getValue(); + if (alb.size() == 1){// when only one column for the family + resultColumnsOnlyFilter.addFilter( + new FilterList(FilterList.Operator.MUST_PASS_ALL, + new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(getName(alb.get(0)))), + new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(getFamily(alb.get(0))))) + ); + }else{// when multiple columns for the family + FamilyFilter familyFilter = null; + FilterList filterListCol = new FilterList(FilterList.Operator.MUST_PASS_ONE); + for(int j = 0; j<alb.size(); j++){ + if (familyFilter == null) + familyFilter = new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(getFamily(alb.get(0)))); + filterListCol.addFilter(new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(getName(alb.get(j))))); + } + resultColumnsOnlyFilter.addFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,filterListCol,familyFilter)); + } + } + } + list.addFilter(resultColumnsOnlyFilter); // add column limiting filter + }//end narrowDownResultColumns if (samplePercent > 0.0f) list.addFilter(new RandomRowFilter(samplePercent)); + // last optimization is making sure we remove top level filter list if it is singleton MUST_PASS_ALL filterlist + if (list.getFilters().size()==1){ + scan.setFilter(list.getFilters().get(0)); + if (logger.isTraceEnabled()) logger.trace("Pushed down filter:"+list.getFilters().get(0)); + }else{ scan.setFilter(list); + if (logger.isTraceEnabled()) logger.trace("Pushed down filter:"+list ); + } } else if (samplePercent > 0.0f) { scan.setFilter(new RandomRowFilter(samplePercent)); }
