HIVE-11110: Reorder applyPreJoinOrderingTransforms, add NotNULL/FilterMerge rules, improve Filter selectivity estimation (Laljo John Pullokkaran reviewed by Jesus Camacho Rodriguez, Ashutosh Chauhan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d8ee05ae Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d8ee05ae Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d8ee05ae Branch: refs/heads/branch-2.0 Commit: d8ee05aeb3eae9540c4aeb685f86eb9a9b9c1d3a Parents: 74fe203 Author: jpullokk <[email protected]> Authored: Mon Dec 14 16:29:44 2015 -0800 Committer: jpullokk <[email protected]> Committed: Mon Dec 14 16:29:44 2015 -0800 ---------------------------------------------------------------------- .../test/results/positive/hbase_queries.q.out | 38 +- .../ql/optimizer/calcite/HiveCalciteUtil.java | 98 + .../calcite/HiveDefaultRelMetadataProvider.java | 2 + .../ql/optimizer/calcite/RelOptHiveTable.java | 9 +- .../hive/ql/optimizer/calcite/TraitsUtil.java | 4 + .../calcite/reloperators/HiveProject.java | 25 +- .../calcite/reloperators/HiveTableScan.java | 46 +- .../rules/HiveFilterAggregateTransposeRule.java | 45 + .../rules/HiveFilterProjectTSTransposeRule.java | 113 + .../HiveJoinPushTransitivePredicatesRule.java | 44 +- .../calcite/rules/HivePartitionPruneRule.java | 2 +- .../calcite/rules/HivePreFilteringRule.java | 170 +- .../stats/FilterSelectivityEstimator.java | 48 + .../calcite/stats/HiveRelMdPredicates.java | 645 ++++ .../translator/SqlFunctionConverter.java | 3 +- .../stats/annotation/StatsRulesProcFactory.java | 62 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 117 +- .../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 8 + .../special_character_in_tabnames_1.q | 26 +- .../bucket_mapjoin_mismatch1.q.out | 28 +- .../sortmerge_mapjoin_mismatch_1.q.out | 10 +- .../clientpositive/allcolref_in_udf.q.out | 14 +- .../results/clientpositive/ambiguous_col.q.out | 54 +- .../clientpositive/annotate_stats_join.q.out | 8 +- .../annotate_stats_join_pkfk.q.out | 61 +- .../archive_excludeHadoop20.q.out | 1 - .../results/clientpositive/archive_multi.q.out | 1 - .../results/clientpositive/auto_join1.q.out | 14 +- .../results/clientpositive/auto_join10.q.out | 12 +- .../results/clientpositive/auto_join12.q.out | 66 +- .../results/clientpositive/auto_join13.q.out | 10 +- .../results/clientpositive/auto_join15.q.out | 12 +- .../results/clientpositive/auto_join16.q.out | 18 +- .../results/clientpositive/auto_join17.q.out | 14 +- .../results/clientpositive/auto_join19.q.out | 14 +- .../results/clientpositive/auto_join2.q.out | 24 +- .../results/clientpositive/auto_join22.q.out | 18 +- .../results/clientpositive/auto_join24.q.out | 10 +- .../results/clientpositive/auto_join26.q.out | 20 +- .../results/clientpositive/auto_join3.q.out | 18 +- .../results/clientpositive/auto_join30.q.out | 64 +- .../results/clientpositive/auto_join33.q.out | 16 +- .../results/clientpositive/auto_join4.q.out | 4 +- .../results/clientpositive/auto_join5.q.out | 4 +- .../results/clientpositive/auto_join8.q.out | 18 +- .../results/clientpositive/auto_join9.q.out | 14 +- .../auto_join_reordering_values.q.out | 10 +- .../clientpositive/auto_join_stats.q.out | 106 +- .../clientpositive/auto_join_stats2.q.out | 46 +- .../auto_join_without_localtask.q.out | 148 +- .../clientpositive/auto_smb_mapjoin_14.q.out | 72 +- .../clientpositive/auto_sortmerge_join_1.q.out | 20 +- .../clientpositive/auto_sortmerge_join_11.q.out | 22 +- .../clientpositive/auto_sortmerge_join_12.q.out | 12 +- .../clientpositive/auto_sortmerge_join_13.q.out | 6 +- .../clientpositive/auto_sortmerge_join_2.q.out | 24 +- .../clientpositive/auto_sortmerge_join_3.q.out | 28 +- .../clientpositive/auto_sortmerge_join_4.q.out | 28 +- .../clientpositive/auto_sortmerge_join_5.q.out | 28 +- .../clientpositive/auto_sortmerge_join_6.q.out | 128 +- .../clientpositive/auto_sortmerge_join_7.q.out | 28 +- .../clientpositive/auto_sortmerge_join_8.q.out | 28 +- .../clientpositive/auto_sortmerge_join_9.q.out | 204 +- .../clientpositive/bucket_map_join_spark1.q.out | 28 +- .../clientpositive/bucket_map_join_spark2.q.out | 28 +- .../clientpositive/bucket_map_join_spark3.q.out | 28 +- .../clientpositive/bucket_map_join_spark4.q.out | 40 +- .../clientpositive/bucketcontext_1.q.out | 8 +- .../clientpositive/bucketcontext_2.q.out | 8 +- .../clientpositive/bucketcontext_3.q.out | 8 +- .../clientpositive/bucketcontext_4.q.out | 8 +- .../clientpositive/bucketcontext_5.q.out | 8 +- .../clientpositive/bucketcontext_6.q.out | 8 +- .../clientpositive/bucketcontext_7.q.out | 8 +- .../clientpositive/bucketcontext_8.q.out | 8 +- .../bucketizedhiveinputformat.q.out | 2 - .../results/clientpositive/bucketmapjoin1.q.out | 20 +- .../clientpositive/bucketmapjoin10.q.out | 6 +- .../clientpositive/bucketmapjoin11.q.out | 12 +- .../clientpositive/bucketmapjoin12.q.out | 12 +- .../clientpositive/bucketmapjoin13.q.out | 24 +- .../results/clientpositive/bucketmapjoin2.q.out | 30 +- .../results/clientpositive/bucketmapjoin3.q.out | 20 +- .../results/clientpositive/bucketmapjoin4.q.out | 20 +- .../results/clientpositive/bucketmapjoin5.q.out | 20 +- .../results/clientpositive/bucketmapjoin7.q.out | 12 +- .../results/clientpositive/bucketmapjoin8.q.out | 12 +- .../results/clientpositive/bucketmapjoin9.q.out | 12 +- .../clientpositive/bucketmapjoin_negative.q.out | 10 +- .../bucketmapjoin_negative2.q.out | 10 +- .../bucketmapjoin_negative3.q.out | 90 +- .../bucketsortoptimize_insert_2.q.out | 56 +- .../bucketsortoptimize_insert_4.q.out | 30 +- .../bucketsortoptimize_insert_5.q.out | 40 +- .../bucketsortoptimize_insert_6.q.out | 114 +- .../bucketsortoptimize_insert_7.q.out | 12 +- .../bucketsortoptimize_insert_8.q.out | 8 +- .../clientpositive/cbo_rp_auto_join1.q.out | 396 ++- .../clientpositive/cbo_rp_auto_join17.q.out | 14 +- .../cbo_rp_cross_product_check_2.q.out | 56 +- .../clientpositive/cbo_rp_lineage2.q.out | 8 +- .../results/clientpositive/cbo_udf_max.q.out | 4 +- .../clientpositive/column_access_stats.q.out | 52 +- .../clientpositive/constprog_partitioner.q.out | 44 +- .../clientpositive/correlationoptimizer1.q.out | 190 +- .../clientpositive/correlationoptimizer10.q.out | 176 +- .../clientpositive/correlationoptimizer11.q.out | 72 +- .../clientpositive/correlationoptimizer12.q.out | 16 +- .../clientpositive/correlationoptimizer13.q.out | 42 +- .../clientpositive/correlationoptimizer14.q.out | 234 +- .../clientpositive/correlationoptimizer15.q.out | 60 +- .../clientpositive/correlationoptimizer2.q.out | 104 +- .../clientpositive/correlationoptimizer3.q.out | 220 +- .../clientpositive/correlationoptimizer4.q.out | 74 +- .../clientpositive/correlationoptimizer5.q.out | 110 +- .../clientpositive/correlationoptimizer6.q.out | 616 ++-- .../clientpositive/correlationoptimizer7.q.out | 104 +- .../clientpositive/correlationoptimizer8.q.out | 34 +- .../clientpositive/correlationoptimizer9.q.out | 120 +- .../results/clientpositive/cross_join.q.out | 28 +- .../clientpositive/cross_product_check_1.q.out | 66 +- .../clientpositive/cross_product_check_2.q.out | 54 +- .../results/clientpositive/decimal_join2.q.out | 36 +- .../results/clientpositive/decimal_udf.q.out | 12 +- .../clientpositive/dynamic_rdd_cache.q.out | 34 +- .../encryption_join_unencrypted_tbl.q.out | 20 +- ...on_join_with_different_encryption_keys.q.out | 16 +- .../clientpositive/explain_logical.q.out | 198 +- .../clientpositive/filter_cond_pushdown.q.out | 241 +- .../clientpositive/filter_join_breaktask.q.out | 32 +- .../clientpositive/groupby_join_pushdown.q.out | 320 +- .../groupby_multi_single_reducer3.q.out | 16 +- .../clientpositive/groupby_position.q.out | 30 +- .../clientpositive/groupby_sort_1_23.q.out | 46 +- .../clientpositive/groupby_sort_skew_1_23.q.out | 50 +- .../test/results/clientpositive/having2.q.out | 66 +- .../identity_project_remove_skip.q.out | 10 +- .../clientpositive/index_auto_mult_tables.q.out | 54 +- .../index_auto_mult_tables_compact.q.out | 42 +- .../clientpositive/index_auto_self_join.q.out | 58 +- .../results/clientpositive/index_bitmap3.q.out | 30 +- .../clientpositive/index_bitmap_auto.q.out | 30 +- .../index_bitmap_auto_partitioned.q.out | 6 +- .../index_bitmap_compression.q.out | 6 +- .../infer_bucket_sort_map_operators.q.out | 22 +- .../test/results/clientpositive/innerjoin.q.out | 18 +- .../test/results/clientpositive/input39.q.out | 10 +- .../results/clientpositive/input_part9.q.out | 4 +- .../clientpositive/input_testxpath2.q.out | 6 +- .../clientpositive/input_testxpath4.q.out | 12 +- ql/src/test/results/clientpositive/join1.q.out | 18 +- ql/src/test/results/clientpositive/join10.q.out | 18 +- ql/src/test/results/clientpositive/join12.q.out | 65 +- ql/src/test/results/clientpositive/join13.q.out | 16 +- ql/src/test/results/clientpositive/join15.q.out | 18 +- ql/src/test/results/clientpositive/join16.q.out | 20 +- ql/src/test/results/clientpositive/join17.q.out | 18 +- ql/src/test/results/clientpositive/join2.q.out | 32 +- ql/src/test/results/clientpositive/join22.q.out | 28 +- ql/src/test/results/clientpositive/join25.q.out | 10 +- ql/src/test/results/clientpositive/join26.q.out | 12 +- ql/src/test/results/clientpositive/join27.q.out | 10 +- ql/src/test/results/clientpositive/join28.q.out | 22 +- ql/src/test/results/clientpositive/join29.q.out | 42 +- ql/src/test/results/clientpositive/join3.q.out | 24 +- ql/src/test/results/clientpositive/join30.q.out | 16 +- ql/src/test/results/clientpositive/join31.q.out | 40 +- ql/src/test/results/clientpositive/join32.q.out | 22 +- .../clientpositive/join32_lessSize.q.out | 124 +- ql/src/test/results/clientpositive/join33.q.out | 22 +- ql/src/test/results/clientpositive/join34.q.out | 4 +- ql/src/test/results/clientpositive/join35.q.out | 4 +- ql/src/test/results/clientpositive/join36.q.out | 10 +- ql/src/test/results/clientpositive/join37.q.out | 10 +- ql/src/test/results/clientpositive/join40.q.out | 32 +- ql/src/test/results/clientpositive/join42.q.out | 77 +- ql/src/test/results/clientpositive/join43.q.out | 66 +- ql/src/test/results/clientpositive/join8.q.out | 16 +- ql/src/test/results/clientpositive/join9.q.out | 18 +- .../clientpositive/join_alt_syntax.q.out | 144 +- .../clientpositive/join_cond_pushdown_1.q.out | 86 +- .../clientpositive/join_cond_pushdown_2.q.out | 74 +- .../clientpositive/join_cond_pushdown_3.q.out | 86 +- .../clientpositive/join_cond_pushdown_4.q.out | 74 +- .../join_cond_pushdown_unqual1.q.out | 24 +- .../join_cond_pushdown_unqual2.q.out | 44 +- .../join_cond_pushdown_unqual3.q.out | 24 +- .../join_cond_pushdown_unqual4.q.out | 44 +- .../clientpositive/join_grp_diff_keys.q.out | 8 +- .../results/clientpositive/join_map_ppr.q.out | 24 +- .../join_merge_multi_expressions.q.out | 20 +- .../results/clientpositive/join_nullsafe.q.out | 36 +- .../clientpositive/join_on_varchar.q.out | 14 +- .../results/clientpositive/join_parse.q.out | 130 +- .../test/results/clientpositive/join_rc.q.out | 18 +- .../results/clientpositive/join_reorder.q.out | 24 +- .../test/results/clientpositive/join_star.q.out | 54 +- .../results/clientpositive/join_thrift.q.out | 14 +- .../test/results/clientpositive/join_vc.q.out | 30 +- .../clientpositive/lateral_view_cp.q.out | 24 +- .../results/clientpositive/limit_pushdown.q.out | 14 +- .../limit_pushdown_negative.q.out | 14 +- .../test/results/clientpositive/lineage2.q.out | 18 +- .../test/results/clientpositive/lineage3.q.out | 20 +- .../list_bucket_query_oneskew_2.q.out | 2 +- .../llap/bucket_map_join_tez1.q.out | 341 +- .../llap/bucket_map_join_tez2.q.out | 135 +- .../llap/dynamic_partition_pruning.q.out | 269 +- .../llap/dynamic_partition_pruning_2.q.out | 21 +- .../llap/hybridgrace_hashjoin_1.q.out | 97 +- .../llap/hybridgrace_hashjoin_2.q.out | 128 +- .../clientpositive/llap/lvj_mapjoin.q.out | 56 +- .../clientpositive/llap/mapjoin_decimal.q.out | 18 +- .../results/clientpositive/llap/mergejoin.q.out | 1614 +++++++-- .../test/results/clientpositive/llap/mrr.q.out | 130 +- .../llap/tez_bmj_schema_evolution.q.out | 18 +- .../llap/tez_dynpart_hashjoin_1.q.out | 120 +- .../llap/tez_dynpart_hashjoin_2.q.out | 118 +- .../clientpositive/llap/tez_join_hash.q.out | 145 +- .../llap/tez_join_result_complex.q.out | 64 +- .../clientpositive/llap/tez_self_join.q.out | 28 +- .../results/clientpositive/llap/tez_smb_1.q.out | 64 +- .../clientpositive/llap/tez_smb_main.q.out | 207 +- .../results/clientpositive/llap/tez_union.q.out | 172 +- .../llap/tez_vector_dynpart_hashjoin_1.q.out | 120 +- .../llap/tez_vector_dynpart_hashjoin_2.q.out | 122 +- .../vectorized_dynamic_partition_pruning.q.out | 269 +- .../llap/vectorized_nested_mapjoin.q.out | 66 +- .../clientpositive/llap_uncompressed.q.out | 6 +- .../clientpositive/louter_join_ppr.q.out | 28 +- .../clientpositive/mapjoin_distinct.q.out | 60 +- .../clientpositive/mapjoin_mapjoin.q.out | 60 +- .../clientpositive/mapjoin_memcheck.q.out | 12 +- .../clientpositive/mapjoin_subquery.q.out | 44 +- .../test/results/clientpositive/mergejoin.q.out | 210 +- .../results/clientpositive/mergejoins.q.out | 38 +- .../clientpositive/mergejoins_mixed.q.out | 266 +- .../results/clientpositive/metadataonly1.q.out | 8 +- .../results/clientpositive/multiMapJoin1.q.out | 384 +-- .../results/clientpositive/multiMapJoin2.q.out | 324 +- .../clientpositive/multi_join_union.q.out | 26 +- .../results/clientpositive/nonmr_fetch.q.out | 16 +- .../test/results/clientpositive/orc_llap.q.out | 52 +- .../clientpositive/orc_predicate_pushdown.q.out | 32 +- .../results/clientpositive/parallel_join1.q.out | 18 +- .../results/clientpositive/parquet_join.q.out | 36 +- .../parquet_predicate_pushdown.q.out | 32 +- ql/src/test/results/clientpositive/pcr.q.out | 40 +- .../results/clientpositive/pointlookup2.q.out | 40 +- .../results/clientpositive/pointlookup3.q.out | 40 +- .../test/results/clientpositive/ppd_gby.q.out | 49 +- .../test/results/clientpositive/ppd_gby2.q.out | 55 +- .../results/clientpositive/ppd_gby_join.q.out | 96 +- .../test/results/clientpositive/ppd_join.q.out | 86 +- .../test/results/clientpositive/ppd_join2.q.out | 133 +- .../test/results/clientpositive/ppd_join3.q.out | 155 +- .../test/results/clientpositive/ppd_join5.q.out | 10 +- .../clientpositive/ppd_join_filter.q.out | 130 +- .../clientpositive/ppd_multi_insert.q.out | 68 +- .../clientpositive/ppd_outer_join2.q.out | 66 +- .../clientpositive/ppd_outer_join3.q.out | 66 +- .../clientpositive/ppd_outer_join4.q.out | 223 +- .../clientpositive/ppd_outer_join5.q.out | 231 +- .../results/clientpositive/ppd_random.q.out | 16 +- .../clientpositive/ppd_repeated_alias.q.out | 2 +- .../results/clientpositive/ppd_udf_case.q.out | 59 +- .../results/clientpositive/ppd_udf_col.q.out | 178 +- .../test/results/clientpositive/ppd_union.q.out | 42 +- .../results/clientpositive/ppd_union_view.q.out | 34 +- ql/src/test/results/clientpositive/ppd_vc.q.out | 30 +- ql/src/test/results/clientpositive/ptf.q.out | 72 +- .../results/clientpositive/ptf_streaming.q.out | 58 +- .../results/clientpositive/ptfgroupbyjoin.q.out | 44 +- .../results/clientpositive/quotedid_basic.q.out | 12 +- .../clientpositive/quotedid_partition.q.out | 4 +- .../reduce_deduplicate_exclude_join.q.out | 12 +- .../reduce_deduplicate_extended.q.out | 86 +- .../test/results/clientpositive/regex_col.q.out | 22 +- .../clientpositive/router_join_ppr.q.out | 30 +- .../runtime_skewjoin_mapjoin_spark.q.out | 58 +- .../test/results/clientpositive/sample8.q.out | 46 +- .../clientpositive/select_transform_hint.q.out | 24 +- .../test/results/clientpositive/semijoin.q.out | 272 +- .../test/results/clientpositive/skewjoin.q.out | 259 +- .../clientpositive/skewjoin_mapjoin10.q.out | 32 +- .../clientpositive/skewjoin_mapjoin6.q.out | 48 +- .../clientpositive/skewjoin_noskew.q.out | 18 +- .../clientpositive/skewjoin_onesideskew.q.out | 18 +- .../results/clientpositive/skewjoinopt10.q.out | 52 +- .../results/clientpositive/skewjoinopt12.q.out | 8 +- .../results/clientpositive/skewjoinopt15.q.out | 40 +- .../results/clientpositive/skewjoinopt16.q.out | 8 +- .../results/clientpositive/skewjoinopt17.q.out | 8 +- .../results/clientpositive/skewjoinopt18.q.out | 14 +- .../results/clientpositive/skewjoinopt2.q.out | 16 +- .../results/clientpositive/smb_mapjoin9.q.out | 2 +- .../results/clientpositive/smb_mapjoin_1.q.out | 2 +- .../results/clientpositive/smb_mapjoin_10.q.out | 2 +- .../results/clientpositive/smb_mapjoin_11.q.out | 2 +- .../results/clientpositive/smb_mapjoin_12.q.out | 4 +- .../results/clientpositive/smb_mapjoin_13.q.out | 14 +- .../results/clientpositive/smb_mapjoin_14.q.out | 18 +- .../results/clientpositive/smb_mapjoin_15.q.out | 18 +- .../results/clientpositive/smb_mapjoin_16.q.out | 2 +- .../results/clientpositive/smb_mapjoin_17.q.out | 2 +- .../results/clientpositive/smb_mapjoin_2.q.out | 4 +- .../results/clientpositive/smb_mapjoin_25.q.out | 105 +- .../results/clientpositive/smb_mapjoin_3.q.out | 2 +- .../results/clientpositive/smb_mapjoin_4.q.out | 2 +- .../results/clientpositive/smb_mapjoin_6.q.out | 4 +- .../clientpositive/sort_merge_join_desc_2.q.out | 2 +- .../clientpositive/sort_merge_join_desc_3.q.out | 2 +- .../clientpositive/sort_merge_join_desc_4.q.out | 6 +- .../clientpositive/sort_merge_join_desc_5.q.out | 2 +- .../clientpositive/sort_merge_join_desc_6.q.out | 6 +- .../clientpositive/sort_merge_join_desc_7.q.out | 6 +- .../clientpositive/sort_merge_join_desc_8.q.out | 6 +- .../spark/annotate_stats_join.q.out | 8 +- .../clientpositive/spark/auto_join1.q.out | 14 +- .../clientpositive/spark/auto_join10.q.out | 12 +- .../clientpositive/spark/auto_join12.q.out | 73 +- .../clientpositive/spark/auto_join13.q.out | 10 +- .../clientpositive/spark/auto_join15.q.out | 12 +- .../clientpositive/spark/auto_join16.q.out | 14 +- .../clientpositive/spark/auto_join17.q.out | 14 +- .../clientpositive/spark/auto_join19.q.out | 14 +- .../clientpositive/spark/auto_join2.q.out | 98 +- .../clientpositive/spark/auto_join22.q.out | 18 +- .../clientpositive/spark/auto_join24.q.out | 10 +- .../clientpositive/spark/auto_join26.q.out | 20 +- .../clientpositive/spark/auto_join3.q.out | 18 +- .../clientpositive/spark/auto_join30.q.out | 38 +- .../clientpositive/spark/auto_join8.q.out | 14 +- .../clientpositive/spark/auto_join9.q.out | 14 +- .../spark/auto_join_reordering_values.q.out | 10 +- .../clientpositive/spark/auto_join_stats.q.out | 58 +- .../clientpositive/spark/auto_join_stats2.q.out | 46 +- .../spark/auto_join_without_localtask.q.out | 68 +- .../spark/auto_smb_mapjoin_14.q.out | 126 +- .../spark/auto_sortmerge_join_1.q.out | 18 +- .../spark/auto_sortmerge_join_12.q.out | 12 +- .../spark/auto_sortmerge_join_13.q.out | 44 +- .../spark/auto_sortmerge_join_2.q.out | 16 +- .../spark/auto_sortmerge_join_3.q.out | 22 +- .../spark/auto_sortmerge_join_4.q.out | 22 +- .../spark/auto_sortmerge_join_5.q.out | 22 +- .../spark/auto_sortmerge_join_6.q.out | 144 +- .../spark/auto_sortmerge_join_7.q.out | 22 +- .../spark/auto_sortmerge_join_8.q.out | 22 +- .../spark/auto_sortmerge_join_9.q.out | 220 +- .../spark/bucket_map_join_spark1.q.out | 28 +- .../spark/bucket_map_join_spark2.q.out | 28 +- .../spark/bucket_map_join_spark3.q.out | 28 +- .../spark/bucket_map_join_spark4.q.out | 140 +- .../spark/bucket_map_join_tez1.q.out | 361 +- .../spark/bucket_map_join_tez2.q.out | 185 +- .../spark/bucketizedhiveinputformat.q.out | 2 - .../clientpositive/spark/bucketmapjoin1.q.out | 20 +- .../clientpositive/spark/bucketmapjoin10.q.out | 6 +- .../clientpositive/spark/bucketmapjoin11.q.out | 12 +- .../clientpositive/spark/bucketmapjoin12.q.out | 12 +- .../clientpositive/spark/bucketmapjoin13.q.out | 24 +- .../clientpositive/spark/bucketmapjoin2.q.out | 30 +- .../clientpositive/spark/bucketmapjoin3.q.out | 20 +- .../clientpositive/spark/bucketmapjoin4.q.out | 20 +- .../clientpositive/spark/bucketmapjoin5.q.out | 20 +- .../clientpositive/spark/bucketmapjoin7.q.out | 12 +- .../clientpositive/spark/bucketmapjoin8.q.out | 12 +- .../clientpositive/spark/bucketmapjoin9.q.out | 12 +- .../spark/bucketmapjoin_negative.q.out | 10 +- .../spark/bucketmapjoin_negative2.q.out | 10 +- .../spark/bucketmapjoin_negative3.q.out | 90 +- .../spark/bucketsortoptimize_insert_2.q.out | 108 +- .../spark/bucketsortoptimize_insert_4.q.out | 30 +- .../spark/bucketsortoptimize_insert_6.q.out | 154 +- .../spark/bucketsortoptimize_insert_7.q.out | 54 +- .../spark/bucketsortoptimize_insert_8.q.out | 36 +- .../spark/column_access_stats.q.out | 52 +- .../spark/constprog_partitioner.q.out | 44 +- .../clientpositive/spark/cross_join.q.out | 28 +- .../spark/cross_product_check_1.q.out | 66 +- .../spark/cross_product_check_2.q.out | 48 +- .../spark/dynamic_rdd_cache.q.out | 34 +- .../spark/filter_join_breaktask.q.out | 28 +- .../spark/groupby_multi_single_reducer3.q.out | 16 +- .../clientpositive/spark/groupby_position.q.out | 32 +- .../spark/groupby_sort_1_23.q.out | 46 +- .../spark/groupby_sort_skew_1_23.q.out | 50 +- .../spark/identity_project_remove_skip.q.out | 8 +- .../spark/index_auto_self_join.q.out | 42 +- .../clientpositive/spark/index_bitmap3.q.out | 30 +- .../spark/index_bitmap_auto.q.out | 30 +- .../spark/infer_bucket_sort_map_operators.q.out | 38 +- .../infer_bucket_sort_reducers_power_two.q.out | 8 +- .../clientpositive/spark/innerjoin.q.out | 18 +- .../results/clientpositive/spark/join1.q.out | 18 +- .../results/clientpositive/spark/join10.q.out | 18 +- .../results/clientpositive/spark/join12.q.out | 51 +- .../results/clientpositive/spark/join13.q.out | 16 +- .../results/clientpositive/spark/join15.q.out | 18 +- .../results/clientpositive/spark/join16.q.out | 20 +- .../results/clientpositive/spark/join17.q.out | 18 +- .../results/clientpositive/spark/join2.q.out | 32 +- .../results/clientpositive/spark/join22.q.out | 22 +- .../results/clientpositive/spark/join25.q.out | 10 +- .../results/clientpositive/spark/join26.q.out | 12 +- .../results/clientpositive/spark/join27.q.out | 10 +- .../results/clientpositive/spark/join28.q.out | 22 +- .../results/clientpositive/spark/join29.q.out | 26 +- .../results/clientpositive/spark/join3.q.out | 24 +- .../results/clientpositive/spark/join30.q.out | 16 +- .../results/clientpositive/spark/join31.q.out | 28 +- .../results/clientpositive/spark/join32.q.out | 22 +- .../clientpositive/spark/join32_lessSize.q.out | 138 +- .../results/clientpositive/spark/join33.q.out | 22 +- .../results/clientpositive/spark/join34.q.out | 6 +- .../results/clientpositive/spark/join35.q.out | 6 +- .../results/clientpositive/spark/join36.q.out | 10 +- .../results/clientpositive/spark/join37.q.out | 10 +- .../results/clientpositive/spark/join40.q.out | 32 +- .../results/clientpositive/spark/join8.q.out | 16 +- .../results/clientpositive/spark/join9.q.out | 18 +- .../clientpositive/spark/join_alt_syntax.q.out | 144 +- .../spark/join_cond_pushdown_1.q.out | 86 +- .../spark/join_cond_pushdown_2.q.out | 74 +- .../spark/join_cond_pushdown_3.q.out | 86 +- .../spark/join_cond_pushdown_4.q.out | 74 +- .../spark/join_cond_pushdown_unqual1.q.out | 24 +- .../spark/join_cond_pushdown_unqual2.q.out | 44 +- .../spark/join_cond_pushdown_unqual3.q.out | 24 +- .../spark/join_cond_pushdown_unqual4.q.out | 44 +- .../clientpositive/spark/join_map_ppr.q.out | 24 +- .../spark/join_merge_multi_expressions.q.out | 20 +- .../clientpositive/spark/join_nullsafe.q.out | 36 +- .../results/clientpositive/spark/join_rc.q.out | 18 +- .../clientpositive/spark/join_reorder.q.out | 24 +- .../clientpositive/spark/join_star.q.out | 54 +- .../clientpositive/spark/join_thrift.q.out | 14 +- .../results/clientpositive/spark/join_vc.q.out | 30 +- .../clientpositive/spark/limit_pushdown.q.out | 14 +- .../clientpositive/spark/louter_join_ppr.q.out | 18 +- .../clientpositive/spark/mapjoin_decimal.q.out | 16 +- .../clientpositive/spark/mapjoin_distinct.q.out | 60 +- .../clientpositive/spark/mapjoin_mapjoin.q.out | 158 +- .../clientpositive/spark/mapjoin_memcheck.q.out | 12 +- .../clientpositive/spark/mapjoin_subquery.q.out | 44 +- .../clientpositive/spark/mergejoins.q.out | 38 +- .../clientpositive/spark/mergejoins_mixed.q.out | 266 +- .../clientpositive/spark/multi_join_union.q.out | 28 +- .../clientpositive/spark/parallel_join1.q.out | 18 +- .../clientpositive/spark/parquet_join.q.out | 42 +- .../test/results/clientpositive/spark/pcr.q.out | 40 +- .../clientpositive/spark/ppd_gby_join.q.out | 96 +- .../results/clientpositive/spark/ppd_join.q.out | 86 +- .../clientpositive/spark/ppd_join2.q.out | 133 +- .../clientpositive/spark/ppd_join3.q.out | 153 +- .../clientpositive/spark/ppd_join5.q.out | 10 +- .../clientpositive/spark/ppd_join_filter.q.out | 130 +- .../clientpositive/spark/ppd_multi_insert.q.out | 68 +- .../clientpositive/spark/ppd_outer_join2.q.out | 66 +- .../clientpositive/spark/ppd_outer_join3.q.out | 66 +- .../clientpositive/spark/ppd_outer_join4.q.out | 211 +- .../clientpositive/spark/ppd_outer_join5.q.out | 207 +- .../test/results/clientpositive/spark/ptf.q.out | 72 +- .../clientpositive/spark/ptf_streaming.q.out | 58 +- .../spark/reduce_deduplicate_exclude_join.q.out | 12 +- .../clientpositive/spark/router_join_ppr.q.out | 20 +- .../spark/runtime_skewjoin_mapjoin_spark.q.out | 26 +- .../results/clientpositive/spark/sample8.q.out | 46 +- .../results/clientpositive/spark/semijoin.q.out | 272 +- .../results/clientpositive/spark/skewjoin.q.out | 283 +- .../clientpositive/spark/skewjoin_noskew.q.out | 18 +- .../clientpositive/spark/skewjoinopt10.q.out | 48 +- .../clientpositive/spark/skewjoinopt12.q.out | 8 +- .../clientpositive/spark/skewjoinopt15.q.out | 32 +- .../clientpositive/spark/skewjoinopt16.q.out | 8 +- .../clientpositive/spark/skewjoinopt17.q.out | 8 +- .../clientpositive/spark/skewjoinopt18.q.out | 14 +- .../clientpositive/spark/skewjoinopt2.q.out | 16 +- .../clientpositive/spark/smb_mapjoin_1.q.out | 14 +- .../clientpositive/spark/smb_mapjoin_10.q.out | 8 +- .../clientpositive/spark/smb_mapjoin_11.q.out | 12 +- .../clientpositive/spark/smb_mapjoin_12.q.out | 24 +- .../clientpositive/spark/smb_mapjoin_13.q.out | 22 +- .../clientpositive/spark/smb_mapjoin_14.q.out | 52 +- .../clientpositive/spark/smb_mapjoin_15.q.out | 42 +- .../clientpositive/spark/smb_mapjoin_16.q.out | 4 +- .../clientpositive/spark/smb_mapjoin_17.q.out | 4 +- .../clientpositive/spark/smb_mapjoin_2.q.out | 16 +- .../clientpositive/spark/smb_mapjoin_25.q.out | 83 +- .../clientpositive/spark/smb_mapjoin_3.q.out | 14 +- .../clientpositive/spark/smb_mapjoin_4.q.out | 8 +- .../clientpositive/spark/smb_mapjoin_5.q.out | 6 +- .../clientpositive/spark/smb_mapjoin_6.q.out | 16 +- .../clientpositive/spark/subquery_exists.q.out | 22 +- .../clientpositive/spark/subquery_in.q.out | 145 +- .../spark/subquery_multiinsert.q.java1.7.out | 14 +- .../spark/table_access_keys_stats.q.out | 37 +- .../clientpositive/spark/temp_table_join1.q.out | 54 +- .../results/clientpositive/spark/union26.q.out | 28 +- .../results/clientpositive/spark/union32.q.out | 80 +- .../results/clientpositive/spark/union34.q.out | 44 +- .../clientpositive/spark/union_top_level.q.out | 16 +- .../spark/vector_decimal_mapjoin.q.out | 12 +- .../spark/vector_mapjoin_reduce.q.out | 166 +- .../spark/vectorization_short_regress.q.out | 18 +- .../spark/vectorized_mapjoin.q.out | 12 +- .../spark/vectorized_nested_mapjoin.q.out | 32 +- .../clientpositive/spark/vectorized_ptf.q.out | 72 +- .../spark/vectorized_shufflejoin.q.out | 16 +- .../test/results/clientpositive/stats11.q.out | 20 +- .../subq_where_serialization.q.out | 30 +- .../clientpositive/subquery_exists.q.out | 22 +- .../clientpositive/subquery_exists_having.q.out | 44 +- .../results/clientpositive/subquery_in.q.out | 145 +- .../clientpositive/subquery_in_having.q.out | 118 +- .../subquery_multiinsert.q.java1.7.out | 14 +- .../results/clientpositive/subquery_notin.q.out | 16 +- .../subquery_notin_having.q.java1.7.out | 6 +- .../subquery_unqual_corr_expr.q.out | 16 +- .../subquery_unqualcolumnrefs.q.out | 142 +- .../results/clientpositive/subquery_views.q.out | 26 +- .../table_access_keys_stats.q.out | 37 +- .../clientpositive/temp_table_join1.q.out | 54 +- .../results/clientpositive/tez/auto_join1.q.out | 16 +- .../clientpositive/tez/auto_join30.q.out | 50 +- .../tez/auto_sortmerge_join_1.q.out | 18 +- .../tez/auto_sortmerge_join_11.q.out | 26 +- .../tez/auto_sortmerge_join_12.q.out | 12 +- .../tez/auto_sortmerge_join_13.q.out | 54 +- .../tez/auto_sortmerge_join_2.q.out | 28 +- .../tez/auto_sortmerge_join_3.q.out | 42 +- .../tez/auto_sortmerge_join_4.q.out | 42 +- .../tez/auto_sortmerge_join_5.q.out | 34 +- .../tez/auto_sortmerge_join_6.q.out | 192 +- .../tez/auto_sortmerge_join_7.q.out | 42 +- .../tez/auto_sortmerge_join_8.q.out | 42 +- .../tez/auto_sortmerge_join_9.q.out | 268 +- .../tez/bucket_map_join_tez1.q.out | 339 +- .../tez/bucket_map_join_tez2.q.out | 133 +- .../tez/correlationoptimizer1.q.out | 208 +- .../results/clientpositive/tez/cross_join.q.out | 30 +- .../tez/cross_product_check_1.q.out | 66 +- .../tez/cross_product_check_2.q.out | 54 +- .../tez/dynamic_partition_pruning.q.out | 228 +- .../tez/dynamic_partition_pruning_2.q.out | 21 +- .../clientpositive/tez/explainuser_1.q.out | 1502 ++++---- .../clientpositive/tez/explainuser_2.q.out | 3195 +++++++++--------- .../clientpositive/tez/explainuser_3.q.out | 34 +- .../tez/filter_join_breaktask.q.out | 28 +- .../tez/hybridgrace_hashjoin_1.q.out | 97 +- .../tez/hybridgrace_hashjoin_2.q.out | 128 +- .../test/results/clientpositive/tez/join1.q.out | 18 +- .../clientpositive/tez/join_nullsafe.q.out | 36 +- .../clientpositive/tez/limit_pushdown.q.out | 14 +- .../clientpositive/tez/lvj_mapjoin.q.out | 56 +- .../clientpositive/tez/mapjoin_decimal.q.out | 18 +- .../clientpositive/tez/mapjoin_mapjoin.q.out | 74 +- .../results/clientpositive/tez/mergejoin.q.out | 204 +- .../test/results/clientpositive/tez/mrr.q.out | 130 +- .../test/results/clientpositive/tez/ptf.q.out | 72 +- .../clientpositive/tez/ptf_streaming.q.out | 58 +- .../results/clientpositive/tez/skewjoin.q.out | 109 +- .../clientpositive/tez/subquery_exists.q.out | 22 +- .../clientpositive/tez/subquery_in.q.out | 145 +- .../tez/tez_bmj_schema_evolution.q.out | 18 +- .../tez/tez_dynpart_hashjoin_1.q.out | 120 +- .../tez/tez_dynpart_hashjoin_2.q.out | 118 +- .../clientpositive/tez/tez_join_hash.q.out | 144 +- .../tez/tez_join_result_complex.q.out | 64 +- .../clientpositive/tez/tez_self_join.q.out | 28 +- .../results/clientpositive/tez/tez_smb_1.q.out | 64 +- .../clientpositive/tez/tez_smb_empty.q.out | 116 +- .../clientpositive/tez/tez_smb_main.q.out | 206 +- .../results/clientpositive/tez/tez_union.q.out | 172 +- .../tez/tez_vector_dynpart_hashjoin_1.q.out | 120 +- .../tez/tez_vector_dynpart_hashjoin_2.q.out | 120 +- .../clientpositive/tez/unionDistinct_1.q.out | 190 +- .../tez/vector_auto_smb_mapjoin_14.q.out | 300 +- .../tez/vector_binary_join_groupby.q.out | 14 +- .../tez/vector_char_mapjoin1.q.out | 56 +- .../tez/vector_decimal_expressions.q.out | 8 +- .../tez/vector_decimal_mapjoin.q.out | 14 +- .../clientpositive/tez/vector_decimal_udf.q.out | 18 +- .../clientpositive/tez/vector_if_expr.q.out | 10 +- .../tez/vector_interval_mapjoin.q.out | 24 +- .../clientpositive/tez/vector_join30.q.out | 50 +- .../tez/vector_join_part_col_char.q.out | 22 +- .../tez/vector_leftsemi_mapjoin.q.out | 1500 ++++---- .../tez/vector_mapjoin_reduce.q.out | 133 +- .../tez/vector_nullsafe_join.q.out | 64 +- .../tez/vector_reduce_groupby_decimal.q.out | 12 +- .../tez/vector_varchar_mapjoin1.q.out | 56 +- .../clientpositive/tez/vectorization_8.q.out | 16 +- .../tez/vectorization_limit.q.out | 22 +- .../tez/vectorization_short_regress.q.out | 18 +- .../tez/vectorized_bucketmapjoin1.q.out | 42 +- .../clientpositive/tez/vectorized_context.q.out | 26 +- .../vectorized_dynamic_partition_pruning.q.out | 228 +- .../clientpositive/tez/vectorized_mapjoin.q.out | 14 +- .../tez/vectorized_nested_mapjoin.q.out | 66 +- .../clientpositive/tez/vectorized_ptf.q.out | 72 +- .../tez/vectorized_shufflejoin.q.out | 16 +- .../udf_case_column_pruning.q.out | 20 +- .../clientpositive/udf_isnull_isnotnull.q.out | 4 +- .../test/results/clientpositive/udf_size.q.out | 4 +- .../clientpositive/udtf_json_tuple.q.out | 26 +- .../clientpositive/udtf_parse_url_tuple.q.out | 22 +- .../test/results/clientpositive/union22.q.out | 6 +- .../test/results/clientpositive/union26.q.out | 34 +- .../test/results/clientpositive/union32.q.out | 96 +- .../test/results/clientpositive/union34.q.out | 56 +- .../clientpositive/unionDistinct_1.q.out | 214 +- .../clientpositive/union_top_level.q.out | 32 +- .../vector_auto_smb_mapjoin_14.q.out | 72 +- .../vector_binary_join_groupby.q.out | 12 +- .../clientpositive/vector_char_mapjoin1.q.out | 50 +- .../clientpositive/vector_decimal_cast.q.out | 4 +- .../vector_decimal_expressions.q.out | 8 +- .../clientpositive/vector_decimal_mapjoin.q.out | 12 +- .../clientpositive/vector_decimal_udf.q.out | 18 +- .../results/clientpositive/vector_if_expr.q.out | 10 +- .../vector_interval_mapjoin.q.out | 22 +- .../results/clientpositive/vector_join30.q.out | 64 +- .../vector_leftsemi_mapjoin.q.out | 1356 ++++---- .../clientpositive/vector_mapjoin_reduce.q.out | 108 +- .../clientpositive/vector_nullsafe_join.q.out | 48 +- .../vector_reduce_groupby_decimal.q.out | 12 +- .../vector_varchar_mapjoin1.q.out | 50 +- .../clientpositive/vectorization_8.q.out | 16 +- .../vectorization_decimal_date.q.out | 4 +- .../clientpositive/vectorization_limit.q.out | 22 +- .../vectorization_short_regress.q.out | 18 +- .../vectorized_bucketmapjoin1.q.out | 6 +- .../clientpositive/vectorized_context.q.out | 22 +- .../clientpositive/vectorized_mapjoin.q.out | 12 +- .../vectorized_nested_mapjoin.q.out | 36 +- .../results/clientpositive/vectorized_ptf.q.out | 72 +- .../clientpositive/vectorized_shufflejoin.q.out | 16 +- 639 files changed, 22891 insertions(+), 20623 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/hbase-handler/src/test/results/positive/hbase_queries.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out index 1ab9877..3907bc9 100644 --- a/hbase-handler/src/test/results/positive/hbase_queries.q.out +++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out @@ -121,7 +121,7 @@ STAGE PLANS: alias: hbase_table_1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: int) @@ -136,17 +136,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Join Operator @@ -156,11 +156,11 @@ STAGE PLANS: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -175,13 +175,13 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 20 Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE @@ -494,19 +494,19 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(key) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -514,7 +514,7 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -529,7 +529,7 @@ STAGE PLANS: alias: hbase_table_1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -546,7 +546,7 @@ STAGE PLANS: key expressions: UDFToDouble(_col0) (type: double) sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -556,14 +556,14 @@ STAGE PLANS: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java index e1b60b0..4825a61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java @@ -25,6 +25,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelOptUtil.InputFinder; import org.apache.calcite.plan.RelOptUtil.InputReferencedVisitor; @@ -32,6 +34,8 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.RelFactories.ProjectFactory; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -44,6 +48,7 @@ import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexRangeRef; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; @@ -613,6 +618,46 @@ public class HiveCalciteUtil { return (new Pair<RelNode, RelNode>(parentOforiginalProjRel, originalProjRel)); } + public static boolean isComparisonOp(RexCall call) { + return call.getKind().belongsTo(SqlKind.COMPARISON); + } + + private static final Function<RexNode, String> REX_STR_FN = new Function<RexNode, String>() { + public String apply(RexNode r) { + return r.toString(); + } + }; + + public static ImmutableList<RexNode> getPredsNotPushedAlready(RelNode inp, List<RexNode> predsToPushDown) { + final RelOptPredicateList predicates = RelMetadataQuery.getPulledUpPredicates(inp); + final ImmutableSet<String> alreadyPushedPreds = ImmutableSet.copyOf(Lists.transform( + predicates.pulledUpPredicates, REX_STR_FN)); + final ImmutableList.Builder<RexNode> newConjuncts = ImmutableList.builder(); + for (RexNode r : predsToPushDown) { + if (!alreadyPushedPreds.contains(r.toString())) { + newConjuncts.add(r); + } + } + return newConjuncts.build(); + } + + public static RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) { + RexNode typeSafeRex = rex; + if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) { + RexBuilder rb = cluster.getRexBuilder(); + List<RexNode> fixedPredElems = new ArrayList<RexNode>(); + RelDataType commonType = cluster.getTypeFactory().leastRestrictive( + RexUtil.types(((RexCall) rex).getOperands())); + for (RexNode rn : ((RexCall) rex).getOperands()) { + fixedPredElems.add(rb.ensureType(commonType, rn, true)); + } + + typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems); + } + + return typeSafeRex; + } + public static boolean isDeterministic(RexNode expr) { boolean deterministic = true; @@ -635,6 +680,58 @@ public class HiveCalciteUtil { return deterministic; } + public static boolean isDeterministicFuncOnLiterals(RexNode expr) { + boolean deterministicFuncOnLiterals = true; + + RexVisitor<Void> visitor = new RexVisitorImpl<Void>(true) { + @Override + public Void visitCall(org.apache.calcite.rex.RexCall call) { + if (!call.getOperator().isDeterministic()) { + throw new Util.FoundOne(call); + } + return super.visitCall(call); + } + + @Override + public Void visitInputRef(RexInputRef inputRef) { + throw new Util.FoundOne(inputRef); + } + + @Override + public Void visitLocalRef(RexLocalRef localRef) { + throw new Util.FoundOne(localRef); + } + + @Override + public Void visitOver(RexOver over) { + throw new Util.FoundOne(over); + } + + @Override + public Void visitDynamicParam(RexDynamicParam dynamicParam) { + throw new Util.FoundOne(dynamicParam); + } + + @Override + public Void visitRangeRef(RexRangeRef rangeRef) { + throw new Util.FoundOne(rangeRef); + } + + @Override + public Void visitFieldAccess(RexFieldAccess fieldAccess) { + throw new Util.FoundOne(fieldAccess); + } + }; + + try { + expr.accept(visitor); + } catch (Util.FoundOne e) { + deterministicFuncOnLiterals = false; + } + + return deterministicFuncOnLiterals; + } + public static <T> ImmutableMap<Integer, T> getColInfoMap(List<T> hiveCols, int startIndx) { Builder<Integer, T> bldr = ImmutableMap.<Integer, T> builder(); @@ -803,6 +900,7 @@ public class HiveCalciteUtil { public static Set<Integer> getInputRefs(RexNode expr) { InputRefsCollector irefColl = new InputRefsCollector(true); + expr.accept(irefColl); return irefColl.getInputRefSet(); } http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java index 748c86d..c0609d7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistinctRowCou import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistribution; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdMemory; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdParallelism; +import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdPredicates; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdRowCount; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSelectivity; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSize; @@ -75,6 +76,7 @@ public class HiveDefaultRelMetadataProvider { new HiveRelMdParallelism(maxSplitSize).getMetadataProvider(), HiveRelMdDistribution.SOURCE, HiveRelMdCollation.SOURCE, + HiveRelMdPredicates.SOURCE, new DefaultRelMetadataProvider())); } http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java index cce3588..02db680 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java @@ -198,7 +198,7 @@ public class RelOptHiveTable extends RelOptAbstractTable { if (null == partitionList) { // we are here either unpartitioned table or partitioned table with no // predicates - computePartitionList(hiveConf, null); + computePartitionList(hiveConf, null, new HashSet<Integer>()); } if (hiveTblMetadata.isPartitioned()) { List<Long> rowCounts = StatsUtils.getBasicStatForPartitions(hiveTblMetadata, @@ -234,8 +234,7 @@ public class RelOptHiveTable extends RelOptAbstractTable { return sb.toString(); } - public void computePartitionList(HiveConf conf, RexNode pruneNode) { - + public void computePartitionList(HiveConf conf, RexNode pruneNode, Set<Integer> partOrVirtualCols) { try { if (!hiveTblMetadata.isPartitioned() || pruneNode == null || InputFinder.bits(pruneNode).length() == 0) { @@ -248,7 +247,7 @@ public class RelOptHiveTable extends RelOptAbstractTable { // We have valid pruning expressions, only retrieve qualifying partitions ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), - HiveCalciteUtil.getInputRefs(pruneNode), this.getRelOptSchema().getTypeFactory())); + partOrVirtualCols, this.getRelOptSchema().getTypeFactory())); partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); @@ -287,7 +286,7 @@ public class RelOptHiveTable extends RelOptAbstractTable { if (null == partitionList) { // We could be here either because its an unpartitioned table or because // there are no pruning predicates on a partitioned table. - computePartitionList(hiveConf, null); + computePartitionList(hiveConf, null, new HashSet<Integer>()); } // 2. Obtain Col Stats for Non Partition Cols http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java index be28828..024ba46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java @@ -34,4 +34,8 @@ public class TraitsUtil { public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster) { return cluster.traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY); } + + public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster, RelTraitSet traitsFromInput) { + return RelTraitSet.createEmpty().merge(traitsFromInput).merge(getDefaultTraitSet(cluster)); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java index 4b7887a..db625c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java @@ -50,6 +50,7 @@ public class HiveProject extends Project implements HiveRelNode { public static final ProjectFactory DEFAULT_PROJECT_FACTORY = new HiveProjectFactoryImpl(); private final List<Integer> virtualCols; + private boolean isSysnthetic; /** * Creates a HiveProject. @@ -165,10 +166,14 @@ public class HiveProject extends Project implements HiveRelNode { } @Override - public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> exps, - RelDataType rowType) { + public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> exps, RelDataType rowType) { assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION); - return new HiveProject(getCluster(), traitSet, input, exps, rowType, getFlags()); + HiveProject hp = new HiveProject(getCluster(), traitSet, input, exps, rowType, getFlags()); + if (this.isSysnthetic()) { + hp.setSynthetic(); + } + + return hp; } @Override @@ -184,6 +189,16 @@ public class HiveProject extends Project implements HiveRelNode { return virtualCols; } + // TODO: this should come through RelBuilder to the constructor as opposed to + // set method. This requires calcite change + public void setSynthetic() { + this.isSysnthetic = true; + } + + public boolean isSysnthetic() { + return isSysnthetic; + } + /** * Implementation of {@link ProjectFactory} that returns * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject} @@ -196,9 +211,9 @@ public class HiveProject extends Project implements HiveRelNode { List<? extends RexNode> childExprs, List<String> fieldNames) { RelOptCluster cluster = child.getCluster(); RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); + RelTraitSet trait = TraitsUtil.getDefaultTraitSet(cluster, child.getTraitSet()); RelNode project = HiveProject.create(cluster, child, - childExprs, rowType, - child.getTraitSet(), Collections.<RelCollation> emptyList()); + childExprs, rowType, trait, Collections.<RelCollation> emptyList()); return project; } http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java index 446dc73..5788805 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java @@ -37,6 +37,7 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Pair; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; @@ -44,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.ColStatistics; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableSet; /** @@ -61,6 +63,7 @@ public class HiveTableScan extends TableScan implements HiveRelNode { private final String tblAlias; private final String concatQbIDAlias; private final boolean useQBIdInDigest; + private final ImmutableSet<Integer> viurtualOrPartColIndxsInTS; public String getTableAlias() { return tblAlias; @@ -94,7 +97,9 @@ public class HiveTableScan extends TableScan implements HiveRelNode { this.tblAlias = alias; this.concatQbIDAlias = concatQbIDAlias; this.hiveTableScanRowType = newRowtype; - this.neededColIndxsFrmReloptHT = buildNeededColIndxsFrmReloptHT(table.getRowType(), newRowtype); + Pair<ImmutableList<Integer>, ImmutableSet<Integer>> colIndxPair = buildColIndxsFrmReloptHT(table, newRowtype); + this.neededColIndxsFrmReloptHT = colIndxPair.getKey(); + this.viurtualOrPartColIndxsInTS = colIndxPair.getValue(); this.useQBIdInDigest = useQBIdInDigest; } @@ -183,7 +188,13 @@ public class HiveTableScan extends TableScan implements HiveRelNode { fieldNames)); // 5. Add Proj on top of TS - return relBuilder.push(newHT).project(exprList, new ArrayList<String>(fieldNames)).build(); + HiveProject hp = (HiveProject) relBuilder.push(newHT) + .project(exprList, new ArrayList<String>(fieldNames)).build(); + + // 6. Set synthetic flag, so that we would push filter below this one + hp.setSynthetic(); + + return hp; } public List<Integer> getNeededColIndxsFrmReloptHT() { @@ -194,17 +205,36 @@ public class HiveTableScan extends TableScan implements HiveRelNode { return hiveTableScanRowType; } - private static ImmutableList<Integer> buildNeededColIndxsFrmReloptHT(RelDataType htRowtype, - RelDataType scanRowType) { + public Set<Integer> getPartOrVirtualCols() { + return viurtualOrPartColIndxsInTS; + } + + private static Pair<ImmutableList<Integer>, ImmutableSet<Integer>> buildColIndxsFrmReloptHT( + RelOptHiveTable relOptHTable, RelDataType scanRowType) { + RelDataType relOptHtRowtype = relOptHTable.getRowType(); + ImmutableList<Integer> neededColIndxsFrmReloptHT; Builder<Integer> neededColIndxsFrmReloptHTBldr = new ImmutableList.Builder<Integer>(); - Map<String, Integer> colNameToPosInReloptHT = HiveCalciteUtil.getRowColNameIndxMap(htRowtype - .getFieldList()); + ImmutableSet<Integer> viurtualOrPartColIndxsInTS; + ImmutableSet.Builder<Integer> viurtualOrPartColIndxsInTSBldr = new ImmutableSet.Builder<Integer>(); + + Map<String, Integer> colNameToPosInReloptHT = HiveCalciteUtil + .getRowColNameIndxMap(relOptHtRowtype.getFieldList()); List<String> colNamesInScanRowType = scanRowType.getFieldNames(); + int partOrVirtualColStartPosInrelOptHtRowtype = relOptHTable.getNonPartColumns().size(); + int tmp; for (int i = 0; i < colNamesInScanRowType.size(); i++) { - neededColIndxsFrmReloptHTBldr.add(colNameToPosInReloptHT.get(colNamesInScanRowType.get(i))); + tmp = colNameToPosInReloptHT.get(colNamesInScanRowType.get(i)); + neededColIndxsFrmReloptHTBldr.add(tmp); + if (tmp >= partOrVirtualColStartPosInrelOptHtRowtype) { + viurtualOrPartColIndxsInTSBldr.add(i); + } } - return neededColIndxsFrmReloptHTBldr.build(); + neededColIndxsFrmReloptHT = neededColIndxsFrmReloptHTBldr.build(); + viurtualOrPartColIndxsInTS = viurtualOrPartColIndxsInTSBldr.build(); + + return new Pair<ImmutableList<Integer>, ImmutableSet<Integer>>(neededColIndxsFrmReloptHT, + viurtualOrPartColIndxsInTS); } } http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java new file mode 100644 index 0000000..0e5c731 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.RelFactories.FilterFactory; +import org.apache.calcite.rel.rules.FilterAggregateTransposeRule; +import org.apache.calcite.rex.RexNode; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; + +public class HiveFilterAggregateTransposeRule extends FilterAggregateTransposeRule { + + public HiveFilterAggregateTransposeRule(Class<? extends Filter> filterClass, + FilterFactory filterFactory, Class<? extends Aggregate> aggregateClass) { + super(filterClass, filterFactory, aggregateClass); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final Filter filterRel = call.rel(0); + RexNode condition = filterRel.getCondition(); + if (!HiveCalciteUtil.isDeterministic(condition)) { + return false; + } + + return super.matches(call); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java new file mode 100644 index 0000000..8321504 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.core.RelFactories.FilterFactory; +import org.apache.calcite.rel.core.RelFactories.ProjectFactory; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexUtil; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; + +//TODO: Remove this once Calcite FilterProjectTransposeRule can take rule operand +public class HiveFilterProjectTSTransposeRule extends RelOptRule { + + private final RelFactories.FilterFactory filterFactory; + private final RelFactories.ProjectFactory projectFactory; + + public HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass, + FilterFactory filterFactory, Class<? extends Project> projectClass, + ProjectFactory projectFactory, Class<? extends TableScan> tsClass) { + super(operand(filterClass, operand(projectClass, operand(tsClass, none())))); + this.filterFactory = filterFactory; + this.projectFactory = projectFactory; + } + + @Override + public boolean matches(RelOptRuleCall call) { + final HiveProject projRel = call.rel(1); + + // Assumption: + // 1. This will be run last after PP, Col Pruning in the PreJoinOrder + // optimizations. + // 2. If ProjectRel is not synthetic then PPD would have already pushed + // relevant pieces down and hence no point in running PPD again. + // 3. For synthetic Projects we don't care about non deterministic UDFs + if (!projRel.isSysnthetic()) { + return false; + } + + return super.matches(call); + } + + // ~ Methods ---------------------------------------------------------------- + + // implement RelOptRule + public void onMatch(RelOptRuleCall call) { + final Filter filter = call.rel(0); + final Project project = call.rel(1); + + if (RexOver.containsOver(project.getProjects(), null)) { + // In general a filter cannot be pushed below a windowing calculation. + // Applying the filter before the aggregation function changes + // the results of the windowing invocation. + // + // When the filter is on the PARTITION BY expression of the OVER clause + // it can be pushed down. For now we don't support this. + return; + } + + if (RexUtil.containsCorrelation(filter.getCondition())) { + // If there is a correlation condition anywhere in the filter, don't + // push this filter past project since in some cases it can prevent a + // Correlate from being de-correlated. + return; + } + + // convert the filter to one that references the child of the project + RexNode newCondition = RelOptUtil.pushPastProject(filter.getCondition(), project); + + // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts + // nullable and not-nullable conditions, but a CAST might get in the way of + // other rewrites. + final RelDataTypeFactory typeFactory = filter.getCluster().getTypeFactory(); + if (RexUtil.isNullabilityCast(typeFactory, newCondition)) { + newCondition = ((RexCall) newCondition).getOperands().get(0); + } + + RelNode newFilterRel = filterFactory == null ? filter.copy(filter.getTraitSet(), + project.getInput(), newCondition) : filterFactory.createFilter(project.getInput(), + newCondition); + + RelNode newProjRel = projectFactory == null ? project.copy(project.getTraitSet(), newFilterRel, + project.getProjects(), project.getRowType()) : projectFactory.createProject(newFilterRel, + project.getProjects(), project.getRowType().getFieldNames()); + + call.transformTo(newProjRel); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java index 29deed9..703c8c6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; import java.util.ArrayList; import java.util.List; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; @@ -26,6 +27,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -35,6 +37,7 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.Util; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; import org.apache.hive.common.util.AnnotationUtils; @@ -68,14 +71,21 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule { @Override public void onMatch(RelOptRuleCall call) { Join join = call.rel(0); + + // Register that we have visited this operator in this rule + HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); + if (registry != null) { + registry.registerVisited(this, join); + } + RelOptPredicateList preds = RelMetadataQuery.getPulledUpPredicates(join); RexBuilder rB = join.getCluster().getRexBuilder(); RelNode lChild = call.rel(1); RelNode rChild = call.rel(2); - List<RexNode> leftPreds = getValidPreds(preds.leftInferredPredicates, lChild.getRowType().getFieldList()); - List<RexNode> rightPreds = getValidPreds(preds.rightInferredPredicates, rChild.getRowType().getFieldList()); + List<RexNode> leftPreds = getValidPreds(join.getCluster(), lChild, preds.leftInferredPredicates, lChild.getRowType()); + List<RexNode> rightPreds = getValidPreds(join.getCluster(), rChild, preds.rightInferredPredicates, rChild.getRowType()); if (leftPreds.isEmpty() && rightPreds.isEmpty()) { return; @@ -97,11 +107,17 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule { lChild, rChild, join.getJoinType(), join.isSemiJoinDone()); call.getPlanner().onCopy(join, newRel); + // We register new Join rel so we do not fire the rule on them again + if (registry != null) { + registry.registerVisited(this, newRel); + } + call.transformTo(newRel); } - private ImmutableList<RexNode> getValidPreds (List<RexNode> rexs, List<RelDataTypeField> types) { - InputRefValidator validator = new InputRefValidator(types); + private ImmutableList<RexNode> getValidPreds(RelOptCluster cluster, RelNode rn, + List<RexNode> rexs, RelDataType rType) { + InputRefValidator validator = new InputRefValidator(rType.getFieldList()); List<RexNode> valids = new ArrayList<RexNode>(rexs.size()); for (RexNode rex : rexs) { try { @@ -111,7 +127,25 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule { Util.swallow(e, null); } } - return ImmutableList.copyOf(valids); + + return HiveCalciteUtil.getPredsNotPushedAlready(rn, valids); + } + + private RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) { + RexNode typeSafeRex = rex; + if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) { + RexBuilder rb = cluster.getRexBuilder(); + List<RexNode> fixedPredElems = new ArrayList<RexNode>(); + RelDataType commonType = cluster.getTypeFactory().leastRestrictive( + RexUtil.types(((RexCall) rex).getOperands())); + for (RexNode rn : ((RexCall) rex).getOperands()) { + fixedPredElems.add(rb.ensureType(commonType, rn, true)); + } + + typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems); + } + + return typeSafeRex; } private static class InputRefValidator extends RexVisitorImpl<Void> { http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java index ba28055..d8c3a22 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java @@ -52,6 +52,6 @@ public class HivePartitionPruneRule extends RelOptRule { Pair<RexNode, RexNode> predicates = PartitionPrune .extractPartitionPredicates(filter.getCluster(), hiveTable, predicate); RexNode partColExpr = predicates.left; - hiveTable.computePartitionList(conf, partColExpr); + hiveTable.computePartitionList(conf, partColExpr, tScan.getPartOrVirtualCols()); } }
