HIVE-13567 : Enable auto-gather column stats by default (Zoltan Haindrich, Pengcheng Xiong via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3bbc24d2 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3bbc24d2 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3bbc24d2 Branch: refs/heads/master Commit: 3bbc24d250b3402491db1c2ac54730dfec22a3bd Parents: 31c1b71 Author: Zoltan Haindrich <[email protected]> Authored: Mon Dec 11 14:03:17 2017 -0800 Committer: Ashutosh Chauhan <[email protected]> Committed: Mon Dec 11 15:19:21 2017 -0800 ---------------------------------------------------------------------- .../accumulo_single_sourced_multi_insert.q.out | 40 +- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../clientpositive/serde_typedbytes.q.out | 30 + .../clientpositive/serde_typedbytes2.q.out | 30 + .../clientpositive/serde_typedbytes3.q.out | 30 + .../clientpositive/serde_typedbytes4.q.out | 24 + .../clientpositive/serde_typedbytes5.q.out | 30 + data/conf/hive-site.xml | 5 +- data/conf/spark/local/hive-site.xml | 6 + data/conf/spark/standalone/hive-site.xml | 6 + data/conf/spark/yarn-client/hive-site.xml | 6 + .../hbase_single_sourced_multi_insert.q.out | 40 +- .../test/results/clientpositive/explain.q.out | 24 + .../insert_into_dynamic_partitions.q.out | 5 + .../clientpositive/insert_into_table.q.out | 68 +- .../insert_overwrite_directory.q.out | 8 +- .../insert_overwrite_dynamic_partitions.q.out | 5 + .../clientpositive/insert_overwrite_table.q.out | 68 +- .../write_final_output_blobstore.q.out | 80 + .../apache/hadoop/hive/ql/TestAcidOnTez.java | 1 + .../clientpositive/alterColumnStatsPart.q | 1 + .../clientpositive/auto_sortmerge_join_13.q | 1 - .../clientpositive/bucket_map_join_tez2.q | 1 + .../clientpositive/bucket_num_reducers.q | 2 +- .../clientpositive/bucket_num_reducers_acid2.q | 1 + .../columnStatsUpdateForStatsOptimizer_1.q | 1 + .../columnStatsUpdateForStatsOptimizer_2.q | 1 + ql/src/test/queries/clientpositive/combine1.q | 2 + .../clientpositive/correlationoptimizer5.q | 2 + .../test/queries/clientpositive/dp_counter_mm.q | 2 + .../clientpositive/encryption_insert_values.q | 3 +- ...yption_join_with_different_encryption_keys.q | 1 + .../clientpositive/encryption_move_tbl.q | 1 + ql/src/test/queries/clientpositive/groupby1.q | 2 + .../queries/clientpositive/groupby1_limit.q | 1 + .../groupby_multi_single_reducer.q | 3 + .../clientpositive/hybridgrace_hashjoin_1.q | 1 + .../infer_bucket_sort_convert_join.q | 9 + .../infer_bucket_sort_reducers_power_two.q | 1 + .../test/queries/clientpositive/input11_limit.q | 1 + .../test/queries/clientpositive/input14_limit.q | 2 + ql/src/test/queries/clientpositive/join2.q | 2 + .../clientpositive/metadata_only_queries.q | 1 + .../metadata_only_queries_with_filters.q | 1 + .../test/queries/clientpositive/multiMapJoin1.q | 1 + .../queries/clientpositive/orc_wide_table.q | 1 + .../clientpositive/partition_coltype_literals.q | 1 + ql/src/test/queries/clientpositive/row__id.q | 2 + .../clientpositive/smb_join_partition_key.q | 2 + .../queries/clientpositive/stats_only_null.q | 1 + ql/src/test/queries/clientpositive/tez_smb_1.q | 1 + .../test/queries/clientpositive/tez_smb_main.q | 1 + .../test/queries/clientpositive/udf_round_2.q | 1 + .../queries/clientpositive/union_remove_26.q | 1 + .../clientpositive/vector_auto_smb_mapjoin_14.q | 1 + .../test/queries/clientpositive/vector_bucket.q | 1 + .../test/queries/clientpositive/vector_char_4.q | 3 +- .../queries/clientpositive/vector_char_simple.q | 1 + .../clientpositive/vector_decimal_mapjoin.q | 2 + .../clientpositive/vector_groupby_rollup1.q | 1 + .../clientpositive/vector_multi_insert.q | 1 + .../clientpositive/vector_tablesample_rows.q | 3 +- .../vector_udf_character_length.q | 1 + .../clientpositive/vector_udf_octet_length.q | 1 + .../queries/clientpositive/vector_varchar_4.q | 3 +- .../clientpositive/vector_varchar_simple.q | 1 + .../vector_windowing_expressions.q | 1 + .../vectorized_insert_into_bucketed_table.q | 3 +- .../clientnegative/fileformat_void_input.q.out | 14 +- .../clientpositive/acid_table_stats.q.out | 4 +- .../clientpositive/alterColumnStats.q.out | 8 +- ...lter_numbuckets_partitioned_table2_h23.q.out | 18 +- ...alter_numbuckets_partitioned_table_h23.q.out | 8 +- .../alter_partition_coltype.q.out | 8 +- .../alter_table_add_partition.q.out | 2 +- .../clientpositive/alter_table_serde2.q.out | 4 +- .../analyze_table_null_partition.q.out | 8 +- .../clientpositive/annotate_stats_filter.q.out | 12 +- .../clientpositive/annotate_stats_groupby.q.out | 34 +- .../clientpositive/array_size_estimation.q.out | 12 +- .../results/clientpositive/auto_join1.q.out | 54 +- .../results/clientpositive/auto_join14.q.out | 54 +- .../results/clientpositive/auto_join17.q.out | 54 +- .../results/clientpositive/auto_join19.q.out | 54 +- .../clientpositive/auto_join19_inclause.q.out | 54 +- .../results/clientpositive/auto_join2.q.out | 54 +- .../results/clientpositive/auto_join25.q.out | 34 +- .../results/clientpositive/auto_join26.q.out | 50 +- .../results/clientpositive/auto_join3.q.out | 54 +- .../results/clientpositive/auto_join4.q.out | 54 +- .../results/clientpositive/auto_join5.q.out | 54 +- .../results/clientpositive/auto_join6.q.out | 44 +- .../results/clientpositive/auto_join7.q.out | 44 +- .../results/clientpositive/auto_join8.q.out | 54 +- .../results/clientpositive/auto_join9.q.out | 54 +- .../auto_join_reordering_values.q.out | 16 +- .../clientpositive/auto_sortmerge_join_13.q.out | 198 +- .../clientpositive/basicstat_partval.q.out | 4 +- .../clientpositive/beeline/smb_mapjoin_11.q.out | 112 +- .../clientpositive/beeline/smb_mapjoin_12.q.out | 115 +- .../clientpositive/beeline/smb_mapjoin_13.q.out | 8 +- .../clientpositive/beeline/smb_mapjoin_7.q.out | 25 + .../clientpositive/binary_output_format.q.out | 50 + .../test/results/clientpositive/bucket1.q.out | 104 +- .../test/results/clientpositive/bucket2.q.out | 40 + .../test/results/clientpositive/bucket3.q.out | 112 +- .../clientpositive/bucket_map_join_spark1.q.out | 232 +- .../clientpositive/bucket_map_join_spark2.q.out | 232 +- .../clientpositive/bucket_map_join_spark3.q.out | 232 +- .../clientpositive/bucket_map_join_spark4.q.out | 24 +- .../clientpositive/bucketmapjoin13.q.out | 18 +- .../results/clientpositive/bucketmapjoin5.q.out | 116 +- .../clientpositive/bucketmapjoin_negative.q.out | 50 + .../bucketmapjoin_negative2.q.out | 50 + .../bucketsortoptimize_insert_1.q.out | 68 +- .../bucketsortoptimize_insert_3.q.out | 56 +- .../bucketsortoptimize_insert_4.q.out | 342 +-- .../bucketsortoptimize_insert_5.q.out | 364 +--- .../bucketsortoptimize_insert_8.q.out | 370 +--- .../clientpositive/case_sensitivity.q.out | 30 + ql/src/test/results/clientpositive/cast1.q.out | 30 + .../cbo_rp_annotate_stats_groupby.q.out | 34 +- .../clientpositive/cbo_rp_auto_join17.q.out | 58 +- .../cbo_rp_gby2_map_multi_distinct.q.out | 96 +- .../cbo_rp_groupby3_noskew_multi_distinct.q.out | 24 + .../results/clientpositive/constprog_dp.q.out | 38 + ql/src/test/results/clientpositive/cp_sel.q.out | 52 +- .../results/clientpositive/deleteAnalyze.q.out | 14 +- .../dynamic_partition_skip_default.q.out | 8 +- .../dynpart_sort_optimization_acid2.q.out | 4 + .../encryption_join_unencrypted_tbl.q.out | 4 +- .../results/clientpositive/explain_ddl.q.out | 30 + .../extrapolate_part_stats_full.q.out | 8 +- .../extrapolate_part_stats_partial.q.out | 88 +- .../clientpositive/filter_join_breaktask.q.out | 64 +- .../test/results/clientpositive/groupby10.q.out | 356 +++- .../test/results/clientpositive/groupby11.q.out | 154 +- .../test/results/clientpositive/groupby12.q.out | 39 +- .../results/clientpositive/groupby1_map.q.out | 44 +- .../clientpositive/groupby1_map_nomap.q.out | 44 +- .../clientpositive/groupby1_map_skew.q.out | 44 +- .../clientpositive/groupby1_noskew.q.out | 39 +- .../results/clientpositive/groupby2_map.q.out | 44 +- .../groupby2_map_multi_distinct.q.out | 88 +- .../clientpositive/groupby2_map_skew.q.out | 44 +- .../clientpositive/groupby2_noskew.q.out | 39 +- .../groupby2_noskew_multi_distinct.q.out | 39 +- .../test/results/clientpositive/groupby3.q.out | 45 +- .../results/clientpositive/groupby3_map.q.out | 24 + .../groupby3_map_multi_distinct.q.out | 24 + .../clientpositive/groupby3_map_skew.q.out | 24 + .../clientpositive/groupby3_noskew.q.out | 20 + .../groupby3_noskew_multi_distinct.q.out | 20 + .../test/results/clientpositive/groupby4.q.out | 62 +- .../results/clientpositive/groupby4_map.q.out | 24 + .../clientpositive/groupby4_map_skew.q.out | 24 + .../clientpositive/groupby4_noskew.q.out | 39 +- .../test/results/clientpositive/groupby5.q.out | 62 +- .../results/clientpositive/groupby5_map.q.out | 24 + .../clientpositive/groupby5_map_skew.q.out | 24 + .../clientpositive/groupby5_noskew.q.out | 39 +- .../test/results/clientpositive/groupby6.q.out | 62 +- .../results/clientpositive/groupby6_map.q.out | 44 +- .../clientpositive/groupby6_map_skew.q.out | 44 +- .../clientpositive/groupby6_noskew.q.out | 39 +- .../results/clientpositive/groupby7_map.q.out | 96 +- .../groupby7_map_multi_single_reducer.q.out | 94 +- .../clientpositive/groupby7_map_skew.q.out | 102 +- .../clientpositive/groupby7_noskew.q.out | 86 +- .../groupby7_noskew_multi_single_reducer.q.out | 88 +- .../test/results/clientpositive/groupby8.q.out | 272 ++- .../results/clientpositive/groupby8_map.q.out | 94 +- .../clientpositive/groupby8_map_skew.q.out | 102 +- .../clientpositive/groupby8_noskew.q.out | 84 +- .../test/results/clientpositive/groupby9.q.out | 480 ++++- .../results/clientpositive/groupby_cube1.q.out | 102 +- .../clientpositive/groupby_cube_multi_gby.q.out | 96 +- .../clientpositive/groupby_map_ppr.q.out | 104 +- .../groupby_map_ppr_multi_distinct.q.out | 104 +- .../groupby_multi_insert_common_distinct.q.out | 96 +- .../groupby_multi_single_reducer2.q.out | 94 +- .../groupby_multi_single_reducer3.q.out | 376 +++- .../clientpositive/groupby_position.q.out | 192 +- .../results/clientpositive/groupby_ppr.q.out | 99 +- .../groupby_ppr_multi_distinct.q.out | 206 +- .../clientpositive/groupby_rollup1.q.out | 102 +- .../clientpositive/groupby_sort_1_23.q.out | 1568 ++++++++++++-- .../results/clientpositive/groupby_sort_2.q.out | 44 +- .../results/clientpositive/groupby_sort_3.q.out | 60 + .../results/clientpositive/groupby_sort_4.q.out | 88 +- .../results/clientpositive/groupby_sort_5.q.out | 104 +- .../results/clientpositive/groupby_sort_6.q.out | 320 ++- .../results/clientpositive/groupby_sort_7.q.out | 30 + .../clientpositive/groupby_sort_skew_1_23.q.out | 1556 ++++++++++++-- .../clientpositive/groupby_sort_test_1.q.out | 30 + .../implicit_cast_during_insert.q.out | 25 + .../clientpositive/index_auto_update.q.out | 38 +- .../clientpositive/infer_bucket_sort.q.out | 50 +- .../infer_bucket_sort_dyn_part.q.out | 68 +- .../infer_bucket_sort_grouping_operators.q.out | 168 +- .../infer_bucket_sort_list_bucket.q.out | 4 +- .../infer_bucket_sort_map_operators.q.out | 176 +- .../infer_bucket_sort_merge.q.out | 4 +- .../infer_bucket_sort_multi_insert.q.out | 8 +- .../infer_bucket_sort_num_buckets.q.out | 56 +- .../test/results/clientpositive/innerjoin.q.out | 44 +- .../test/results/clientpositive/input11.q.out | 30 + .../test/results/clientpositive/input12.q.out | 200 +- .../test/results/clientpositive/input13.q.out | 244 ++- .../test/results/clientpositive/input14.q.out | 44 +- .../test/results/clientpositive/input17.q.out | 44 +- .../test/results/clientpositive/input18.q.out | 44 +- .../results/clientpositive/input1_limit.q.out | 98 +- .../test/results/clientpositive/input20.q.out | 44 +- .../test/results/clientpositive/input30.q.out | 48 + .../test/results/clientpositive/input31.q.out | 24 + .../test/results/clientpositive/input32.q.out | 24 + .../test/results/clientpositive/input33.q.out | 44 +- .../test/results/clientpositive/input34.q.out | 30 + .../test/results/clientpositive/input35.q.out | 30 + .../test/results/clientpositive/input36.q.out | 30 + .../test/results/clientpositive/input38.q.out | 30 + .../results/clientpositive/input3_limit.q.out | 45 +- ql/src/test/results/clientpositive/input5.q.out | 44 +- ql/src/test/results/clientpositive/input6.q.out | 30 + ql/src/test/results/clientpositive/input7.q.out | 30 + ql/src/test/results/clientpositive/input8.q.out | 30 + ql/src/test/results/clientpositive/input9.q.out | 30 + .../results/clientpositive/input_part1.q.out | 50 + .../results/clientpositive/input_part10.q.out | 53 +- .../results/clientpositive/input_part2.q.out | 314 ++- .../results/clientpositive/input_part5.q.out | 30 + .../clientpositive/input_testsequencefile.q.out | 30 + .../clientpositive/input_testxpath.q.out | 30 + .../clientpositive/input_testxpath2.q.out | 30 + .../test/results/clientpositive/insert1.q.out | 232 +- .../insert1_overwrite_partitions.q.out | 98 +- .../insert2_overwrite_partitions.q.out | 106 +- .../results/clientpositive/insert_into1.q.out | 195 +- .../results/clientpositive/insert_into2.q.out | 159 +- .../results/clientpositive/insert_into3.q.out | 196 +- .../results/clientpositive/insert_into4.q.out | 120 +- .../results/clientpositive/insert_into5.q.out | 151 +- .../results/clientpositive/insert_into6.q.out | 91 +- .../clientpositive/insertoverwrite_bucket.q.out | 88 +- ql/src/test/results/clientpositive/join14.q.out | 44 +- ql/src/test/results/clientpositive/join17.q.out | 104 +- ql/src/test/results/clientpositive/join25.q.out | 54 +- ql/src/test/results/clientpositive/join26.q.out | 114 +- ql/src/test/results/clientpositive/join27.q.out | 54 +- ql/src/test/results/clientpositive/join28.q.out | 54 +- ql/src/test/results/clientpositive/join29.q.out | 96 +- ql/src/test/results/clientpositive/join3.q.out | 44 +- ql/src/test/results/clientpositive/join30.q.out | 50 +- ql/src/test/results/clientpositive/join31.q.out | 50 +- ql/src/test/results/clientpositive/join32.q.out | 114 +- ql/src/test/results/clientpositive/join33.q.out | 114 +- ql/src/test/results/clientpositive/join34.q.out | 141 +- ql/src/test/results/clientpositive/join35.q.out | 145 +- ql/src/test/results/clientpositive/join36.q.out | 54 +- ql/src/test/results/clientpositive/join37.q.out | 54 +- ql/src/test/results/clientpositive/join39.q.out | 54 +- ql/src/test/results/clientpositive/join4.q.out | 44 +- ql/src/test/results/clientpositive/join5.q.out | 44 +- ql/src/test/results/clientpositive/join6.q.out | 44 +- ql/src/test/results/clientpositive/join7.q.out | 44 +- ql/src/test/results/clientpositive/join8.q.out | 44 +- ql/src/test/results/clientpositive/join9.q.out | 104 +- .../results/clientpositive/join_map_ppr.q.out | 116 +- .../results/clientpositive/lb_fs_stats.q.out | 2 +- .../limit_pushdown_negative.q.out | 97 +- .../test/results/clientpositive/lineage1.q.out | 43 + .../clientpositive/list_bucket_dml_1.q.out | 64 +- .../clientpositive/list_bucket_dml_11.q.out | 62 +- .../clientpositive/list_bucket_dml_12.q.out | 64 +- .../clientpositive/list_bucket_dml_13.q.out | 62 +- .../clientpositive/list_bucket_dml_14.q.out | 56 +- .../clientpositive/list_bucket_dml_2.q.out | 62 +- .../clientpositive/list_bucket_dml_3.q.out | 62 +- .../clientpositive/list_bucket_dml_4.q.out | 122 +- .../clientpositive/list_bucket_dml_5.q.out | 66 +- .../clientpositive/list_bucket_dml_6.q.out | 128 +- .../clientpositive/list_bucket_dml_7.q.out | 128 +- .../clientpositive/list_bucket_dml_8.q.out | 64 +- .../clientpositive/list_bucket_dml_9.q.out | 122 +- .../list_bucket_query_multiskew_1.q.out | 10 +- .../list_bucket_query_multiskew_2.q.out | 8 +- .../list_bucket_query_multiskew_3.q.out | 12 +- .../llap/acid_bucket_pruning.q.out | 2 - .../clientpositive/llap/acid_no_buckets.q.out | 47 +- .../llap/acid_vectorization_original.q.out | 36 +- .../llap/alter_merge_stats_orc.q.out | 6 +- .../clientpositive/llap/auto_join1.q.out | 33 + .../llap/auto_smb_mapjoin_14.q.out | 474 +++-- .../llap/auto_sortmerge_join_10.q.out | 62 +- .../llap/auto_sortmerge_join_13.q.out | 276 ++- .../llap/auto_sortmerge_join_14.q.out | 40 +- .../llap/auto_sortmerge_join_15.q.out | 40 +- .../llap/auto_sortmerge_join_6.q.out | 762 +++---- .../llap/auto_sortmerge_join_9.q.out | 652 +++--- .../results/clientpositive/llap/bucket2.q.out | 40 + .../results/clientpositive/llap/bucket3.q.out | 61 + .../results/clientpositive/llap/bucket4.q.out | 40 + .../results/clientpositive/llap/bucket5.q.out | 110 +- .../results/clientpositive/llap/bucket6.q.out | 33 + .../clientpositive/llap/bucket_groupby.q.out | 198 +- .../clientpositive/llap/bucket_many.q.out | 53 + .../llap/bucket_map_join_tez1.q.out | 28 +- .../clientpositive/llap/bucketmapjoin1.q.out | 118 +- .../clientpositive/llap/bucketmapjoin2.q.out | 179 +- .../clientpositive/llap/bucketmapjoin3.q.out | 118 +- .../clientpositive/llap/bucketmapjoin4.q.out | 118 +- .../llap/bucketsortoptimize_insert_2.q.out | 390 +++- .../llap/bucketsortoptimize_insert_6.q.out | 455 +++- .../llap/bucketsortoptimize_insert_7.q.out | 195 +- .../clientpositive/llap/cbo_rp_lineage2.q.out | 18 +- .../llap/column_access_stats.q.out | 62 +- .../llap/column_table_stats_orc.q.out | 34 +- .../llap/columnstats_part_coltype.q.out | 112 +- .../llap/constprog_semijoin.q.out | 158 +- .../clientpositive/llap/cross_prod_3.q.out | 20 +- .../results/clientpositive/llap/cte_5.q.out | 6 +- .../results/clientpositive/llap/cte_mat_5.q.out | 2 +- .../clientpositive/llap/deleteAnalyze.q.out | 14 +- .../llap/disable_merge_for_bucketing.q.out | 52 +- .../clientpositive/llap/dp_counter_non_mm.q.out | 38 +- .../llap/dynamic_semijoin_reduction_3.q.out | 207 +- .../llap/dynamic_semijoin_reduction_sw.q.out | 34 +- .../llap/dynpart_sort_opt_vectorization.q.out | 417 ++-- .../llap/dynpart_sort_optimization.q.out | 270 ++- .../llap/dynpart_sort_optimization2.q.out | 258 ++- .../clientpositive/llap/except_distinct.q.out | 48 +- .../clientpositive/llap/explainanalyze_2.q.out | 1481 +++++++------ .../clientpositive/llap/explainuser_1.q.out | 351 +-- .../clientpositive/llap/explainuser_2.q.out | 1707 ++++++++------- .../extrapolate_part_stats_partial_ndv.q.out | 22 +- .../llap/filter_join_breaktask.q.out | 90 +- .../results/clientpositive/llap/groupby2.q.out | 42 + .../results/clientpositive/llap/groupby3.q.out | 34 + .../llap/infer_bucket_sort_bucketed_table.q.out | 2 +- .../results/clientpositive/llap/insert1.q.out | 208 ++ .../clientpositive/llap/insert_into1.q.out | 172 +- .../clientpositive/llap/insert_into2.q.out | 126 +- .../insert_values_orig_table_use_metadata.q.out | 7 +- .../clientpositive/llap/intersect_all.q.out | 44 +- .../llap/intersect_distinct.q.out | 40 +- .../clientpositive/llap/intersect_merge.q.out | 540 ++--- .../results/clientpositive/llap/join1.q.out | 37 +- .../clientpositive/llap/join32_lessSize.q.out | 356 +++- .../results/clientpositive/llap/join46.q.out | 326 +-- .../llap/join_emit_interval.q.out | 32 +- .../results/clientpositive/llap/lineage2.q.out | 20 +- .../results/clientpositive/llap/lineage3.q.out | 16 +- .../llap/list_bucket_dml_10.q.out | 65 +- .../clientpositive/llap/llap_partitioned.q.out | 8 +- .../results/clientpositive/llap/llap_smb.q.out | 18 +- .../clientpositive/llap/llap_stats.q.out | 4 +- .../clientpositive/llap/load_dyn_part1.q.out | 84 + .../clientpositive/llap/load_dyn_part2.q.out | 41 + .../clientpositive/llap/load_dyn_part3.q.out | 43 + .../clientpositive/llap/load_dyn_part5.q.out | 43 + .../results/clientpositive/llap/mapjoin3.q.out | 16 +- .../results/clientpositive/llap/mapjoin46.q.out | 280 +-- .../clientpositive/llap/mapjoin_decimal.q.out | 22 +- .../llap/mapjoin_emit_interval.q.out | 28 +- .../clientpositive/llap/mapreduce1.q.out | 33 + .../clientpositive/llap/mapreduce2.q.out | 33 + .../results/clientpositive/llap/merge1.q.out | 115 +- .../results/clientpositive/llap/merge2.q.out | 115 +- .../results/clientpositive/llap/mergejoin.q.out | 476 +++-- .../results/clientpositive/llap/mm_all.q.out | 53 +- .../clientpositive/llap/multiMapJoin2.q.out | 16 +- .../llap/multi_count_distinct_null.q.out | 42 +- .../clientpositive/llap/multi_insert.q.out | 920 +++++++- .../llap/multi_insert_lateral_view.q.out | 502 ++++- .../clientpositive/llap/optimize_join_ptp.q.out | 22 +- .../clientpositive/llap/orc_analyze.q.out | 10 +- .../clientpositive/llap/orc_merge1.q.out | 129 ++ .../clientpositive/llap/orc_merge10.q.out | 129 ++ .../clientpositive/llap/orc_merge2.q.out | 43 + .../clientpositive/llap/orc_merge3.q.out | 41 +- .../clientpositive/llap/orc_merge4.q.out | 41 +- .../clientpositive/llap/orc_merge5.q.out | 70 + .../clientpositive/llap/orc_merge6.q.out | 86 + .../clientpositive/llap/orc_merge7.q.out | 86 + .../clientpositive/llap/orc_merge_diff_fs.q.out | 129 ++ .../llap/orc_merge_incompat1.q.out | 35 + .../llap/orc_merge_incompat2.q.out | 43 + .../llap/orc_predicate_pushdown.q.out | 144 +- .../results/clientpositive/llap/parallel.q.out | 66 + .../llap/parquet_predicate_pushdown.q.out | 136 +- .../clientpositive/llap/partition_pruning.q.out | 10 +- .../llap/partition_shared_scan.q.out | 46 +- .../clientpositive/llap/ppd_union_view.q.out | 30 +- .../test/results/clientpositive/llap/ptf.q.out | 72 +- .../clientpositive/llap/rcfile_createas1.q.out | 6 +- .../clientpositive/llap/rcfile_merge2.q.out | 43 + .../clientpositive/llap/rcfile_merge3.q.out | 41 +- .../clientpositive/llap/rcfile_merge4.q.out | 41 +- .../llap/reduce_deduplicate.q.out | 101 + .../llap/reduce_deduplicate_distinct.q.out | 84 +- .../results/clientpositive/llap/sample1.q.out | 55 + .../results/clientpositive/llap/sample10.q.out | 26 +- .../llap/schema_evol_orc_nonvec_part.q.out | 48 +- ...chema_evol_orc_nonvec_part_all_complex.q.out | 6 +- ...ema_evol_orc_nonvec_part_all_primitive.q.out | 24 +- .../llap/schema_evol_orc_nonvec_table.q.out | 30 +- .../llap/schema_evol_orc_vec_part.q.out | 48 +- .../schema_evol_orc_vec_part_all_complex.q.out | 6 +- ...schema_evol_orc_vec_part_all_primitive.q.out | 24 +- .../llap/schema_evol_orc_vec_table.q.out | 30 +- .../llap/schema_evol_text_nonvec_part.q.out | 48 +- ...hema_evol_text_nonvec_part_all_complex.q.out | 6 +- ...ma_evol_text_nonvec_part_all_primitive.q.out | 24 +- .../llap/schema_evol_text_nonvec_table.q.out | 30 +- .../llap/schema_evol_text_vec_part.q.out | 48 +- .../schema_evol_text_vec_part_all_complex.q.out | 6 +- ...chema_evol_text_vec_part_all_primitive.q.out | 24 +- .../llap/schema_evol_text_vec_table.q.out | 30 +- .../llap/schema_evol_text_vecrow_part.q.out | 48 +- ...hema_evol_text_vecrow_part_all_complex.q.out | 6 +- ...ma_evol_text_vecrow_part_all_primitive.q.out | 24 +- .../llap/schema_evol_text_vecrow_table.q.out | 30 +- .../results/clientpositive/llap/semijoin6.q.out | 148 +- .../results/clientpositive/llap/semijoin7.q.out | 136 +- .../results/clientpositive/llap/skewjoin.q.out | 37 +- .../clientpositive/llap/skewjoinopt15.q.out | 84 +- .../results/clientpositive/llap/smb_cache.q.out | 36 +- .../clientpositive/llap/smb_mapjoin_14.q.out | 332 +-- .../clientpositive/llap/smb_mapjoin_15.q.out | 136 +- .../clientpositive/llap/smb_mapjoin_17.q.out | 242 +-- .../clientpositive/llap/smb_mapjoin_18.q.out | 73 +- .../clientpositive/llap/smb_mapjoin_19.q.out | 10 +- .../clientpositive/llap/smb_mapjoin_6.q.out | 256 ++- .../results/clientpositive/llap/sqlmerge.q.out | 98 +- .../results/clientpositive/llap/stats11.q.out | 118 +- .../clientpositive/llap/subquery_exists.q.out | 66 +- .../clientpositive/llap/subquery_in.q.out | 38 +- .../clientpositive/llap/subquery_multi.q.out | 34 +- .../clientpositive/llap/subquery_notin.q.out | 370 ++-- .../clientpositive/llap/subquery_scalar.q.out | 204 +- .../clientpositive/llap/subquery_select.q.out | 32 +- .../llap/tez_bmj_schema_evolution.q.out | 22 +- .../results/clientpositive/llap/tez_dml.q.out | 111 + .../clientpositive/llap/tez_fsstat.q.out | 2 +- .../clientpositive/llap/tez_join_hash.q.out | 18 +- .../clientpositive/llap/tez_nway_join.q.out | 122 +- .../clientpositive/llap/tez_self_join.q.out | 114 +- .../clientpositive/llap/tez_smb_empty.q.out | 52 +- .../llap/tez_union_dynamic_partition.q.out | 61 +- .../llap/tez_union_dynamic_partition_2.q.out | 61 +- .../llap/tez_union_multiinsert.q.out | 370 +++- .../results/clientpositive/llap/union4.q.out | 52 +- .../results/clientpositive/llap/union6.q.out | 50 +- .../clientpositive/llap/unionDistinct_1.q.out | 1648 ++++++++++----- .../clientpositive/llap/unionDistinct_3.q.out | 37 +- .../clientpositive/llap/union_stats.q.out | 4 +- .../clientpositive/llap/union_top_level.q.out | 148 +- .../llap/vector_adaptor_usage_mode.q.out | 100 +- .../llap/vector_aggregate_without_gby.q.out | 14 +- .../llap/vector_binary_join_groupby.q.out | 68 +- .../clientpositive/llap/vector_bround.q.out | 6 +- .../llap/vector_cast_constant.q.out | 18 +- .../clientpositive/llap/vector_char_2.q.out | 36 +- .../clientpositive/llap/vector_coalesce_2.q.out | 40 +- .../clientpositive/llap/vector_coalesce_3.q.out | 16 +- .../llap/vector_complex_join.q.out | 8 +- .../llap/vector_count_distinct.q.out | 18 +- .../clientpositive/llap/vector_data_types.q.out | 24 +- .../clientpositive/llap/vector_decimal_1.q.out | 90 +- .../clientpositive/llap/vector_decimal_2.q.out | 160 +- .../clientpositive/llap/vector_decimal_6.q.out | 46 +- .../llap/vector_decimal_precision.q.out | 12 +- .../llap/vector_decimal_round.q.out | 60 +- .../llap/vector_decimal_round_2.q.out | 40 +- .../llap/vector_decimal_trailing.q.out | 10 +- .../llap/vector_decimal_udf.q.out | 250 +-- .../llap/vector_decimal_udf2.q.out | 16 +- .../clientpositive/llap/vector_groupby4.q.out | 60 + .../clientpositive/llap/vector_groupby6.q.out | 60 + .../llap/vector_groupby_cube1.q.out | 72 +- .../llap/vector_groupby_grouping_sets2.q.out | 18 +- .../llap/vector_groupby_grouping_window.q.out | 22 +- .../llap/vector_groupby_reduce.q.out | 84 +- .../clientpositive/llap/vector_inner_join.q.out | 178 +- .../llap/vector_left_outer_join2.q.out | 96 +- .../llap/vector_outer_join0.q.out | 28 +- .../llap/vector_partition_diff_num_cols.q.out | 60 +- .../llap/vector_partitioned_date_time.q.out | 48 +- .../llap/vector_ptf_part_simple.q.out | 312 +-- .../llap/vector_string_concat.q.out | 8 +- .../clientpositive/llap/vector_struct_in.q.out | 56 +- .../clientpositive/llap/vector_udf1.q.out | 220 +- .../llap/vector_when_case_null.q.out | 12 +- .../clientpositive/llap/vector_windowing.q.out | 130 +- .../llap/vector_windowing_windowspec4.q.out | 12 +- .../vectorization_input_format_excludes.q.out | 88 +- .../llap/vectorization_part_project.q.out | 12 +- .../llap/vectorized_bucketmapjoin1.q.out | 54 +- .../clientpositive/llap/vectorized_case.q.out | 36 +- .../llap/vectorized_context.q.out | 65 +- .../clientpositive/llap/vectorized_join46.q.out | 250 +-- .../llap/vectorized_parquet.q.out | 12 +- .../llap/vectorized_parquet_types.q.out | 16 +- .../clientpositive/llap/vectorized_ptf.q.out | 620 +++--- .../llap/vectorized_timestamp.q.out | 54 +- .../llap/vectorized_timestamp_funcs.q.out | 80 +- .../test/results/clientpositive/llap_acid.q.out | 76 +- .../results/clientpositive/llap_acid_fast.q.out | 76 +- .../results/clientpositive/load_dyn_part1.q.out | 120 +- .../clientpositive/load_dyn_part10.q.out | 38 + .../clientpositive/load_dyn_part13.q.out | 54 + .../clientpositive/load_dyn_part14.q.out | 73 +- .../results/clientpositive/load_dyn_part2.q.out | 52 +- .../results/clientpositive/load_dyn_part3.q.out | 38 + .../results/clientpositive/load_dyn_part4.q.out | 38 + .../results/clientpositive/load_dyn_part8.q.out | 165 +- .../results/clientpositive/load_dyn_part9.q.out | 38 + .../results/clientpositive/mapjoin_hook.q.out | 36 +- .../results/clientpositive/mapreduce1.q.out | 44 +- .../results/clientpositive/mapreduce2.q.out | 44 +- .../results/clientpositive/mapreduce3.q.out | 44 +- .../results/clientpositive/mapreduce4.q.out | 44 +- .../results/clientpositive/mapreduce5.q.out | 44 +- .../results/clientpositive/mapreduce6.q.out | 44 +- .../results/clientpositive/mapreduce7.q.out | 44 +- .../results/clientpositive/mapreduce8.q.out | 44 +- ql/src/test/results/clientpositive/merge1.q.out | 104 +- ql/src/test/results/clientpositive/merge2.q.out | 104 +- ql/src/test/results/clientpositive/merge3.q.out | 112 +- ql/src/test/results/clientpositive/merge4.q.out | 130 +- .../merge_dynamic_partition.q.out | 114 + .../merge_dynamic_partition2.q.out | 38 + .../merge_dynamic_partition3.q.out | 38 + .../merge_dynamic_partition4.q.out | 38 + .../merge_dynamic_partition5.q.out | 38 + ql/src/test/results/clientpositive/mm_all.q.out | 42 +- .../clientpositive/multi_insert_gby.q.out | 188 +- .../clientpositive/multi_insert_gby2.q.out | 48 + .../clientpositive/multi_insert_gby3.q.out | 434 +++- .../clientpositive/multi_insert_gby4.q.out | 144 +- .../clientpositive/multi_insert_mixed.q.out | 96 +- ...i_insert_move_tasks_share_dependencies.q.out | 1998 +++++++++++++++--- .../clientpositive/multi_insert_union_src.q.out | 48 + .../multi_insert_with_join2.q.out | 652 +++++- .../clientpositive/multigroupby_singlemr.q.out | 534 ++++- .../nonreserved_keywords_insert_into1.q.out | 135 +- .../results/clientpositive/notable_alias1.q.out | 44 +- .../results/clientpositive/notable_alias2.q.out | 44 +- .../results/clientpositive/orc_merge1.q.out | 114 + .../results/clientpositive/orc_merge10.q.out | 114 + .../results/clientpositive/orc_merge2.q.out | 38 + .../results/clientpositive/orc_merge3.q.out | 30 + .../results/clientpositive/orc_merge4.q.out | 30 + .../results/clientpositive/orc_merge5.q.out | 60 + .../results/clientpositive/orc_merge6.q.out | 76 + .../clientpositive/orc_merge_diff_fs.q.out | 114 + .../clientpositive/orc_merge_incompat1.q.out | 30 + .../clientpositive/orc_merge_incompat2.q.out | 38 + .../test/results/clientpositive/parallel.q.out | 94 +- .../results/clientpositive/parallel_join1.q.out | 44 +- .../clientpositive/parquet_analyze.q.out | 4 +- ql/src/test/results/clientpositive/pcr.q.out | 716 +++---- .../results/clientpositive/pointlookup2.q.out | 28 +- .../results/clientpositive/pointlookup3.q.out | 18 +- .../results/clientpositive/pointlookup4.q.out | 8 +- .../clientpositive/ppd_constant_expr.q.out | 60 + .../clientpositive/ppd_multi_insert.q.out | 304 ++- .../test/results/clientpositive/push_or.q.out | 4 +- ql/src/test/results/clientpositive/quote1.q.out | 38 + .../clientpositive/rand_partitionpruner2.q.out | 50 + .../clientpositive/rcfile_default_format.q.out | 2 +- .../clientpositive/rcfile_null_value.q.out | 44 +- .../test/results/clientpositive/sample1.q.out | 50 + .../test/results/clientpositive/sample2.q.out | 50 + .../test/results/clientpositive/sample4.q.out | 50 + .../test/results/clientpositive/sample5.q.out | 50 + .../test/results/clientpositive/sample6.q.out | 50 + .../test/results/clientpositive/sample7.q.out | 50 + .../test/results/clientpositive/skewjoin.q.out | 73 +- .../results/clientpositive/smb_mapjoin_11.q.out | 112 +- .../results/clientpositive/smb_mapjoin_12.q.out | 115 +- .../results/clientpositive/smb_mapjoin_13.q.out | 8 +- .../results/clientpositive/smb_mapjoin_20.q.out | 156 +- .../results/clientpositive/smb_mapjoin_21.q.out | 264 ++- .../results/clientpositive/smb_mapjoin_22.q.out | 8 + .../results/clientpositive/smb_mapjoin_7.q.out | 25 + .../clientpositive/sort_merge_join_desc_5.q.out | 2 +- .../clientpositive/sort_merge_join_desc_6.q.out | 4 +- .../clientpositive/sort_merge_join_desc_7.q.out | 8 +- ql/src/test/results/clientpositive/stats0.q.out | 176 ++ ql/src/test/results/clientpositive/stats1.q.out | 45 +- .../test/results/clientpositive/stats10.q.out | 56 +- .../test/results/clientpositive/stats14.q.out | 6 +- .../test/results/clientpositive/stats15.q.out | 6 +- .../test/results/clientpositive/stats18.q.out | 2 +- ql/src/test/results/clientpositive/stats4.q.out | 128 +- .../clientpositive/stats_empty_dyn_part.q.out | 38 + .../clientpositive/stats_empty_partition.q.out | 2 +- .../clientpositive/stats_list_bucket.q.out | 4 +- .../clientpositive/stats_partial_size.q.out | 10 +- .../test/results/clientpositive/statsfs.q.out | 10 +- .../clientpositive/subquery_multiinsert.q.out | 198 +- .../tez/acid_vectorization_original_tez.q.out | 36 +- .../clientpositive/tez/explainanalyze_1.q.out | 25 +- .../clientpositive/tez/explainanalyze_3.q.out | 66 +- .../clientpositive/tez/explainuser_3.q.out | 70 +- .../tez/multi_count_distinct.q.out | 34 +- .../results/clientpositive/tez/tez-tag.q.out | 20 +- .../tez/vector_join_part_col_char.q.out | 20 +- .../tez/vector_non_string_partition.q.out | 28 +- .../clientpositive/truncate_column.q.out | 6 +- ql/src/test/results/clientpositive/udf1.q.out | 30 + ql/src/test/results/clientpositive/udf3.q.out | 24 + .../results/clientpositive/udf_10_trims.q.out | 30 + .../clientpositive/udf_character_length.q.out | 30 + .../results/clientpositive/udf_length.q.out | 30 + .../clientpositive/udf_octet_length.q.out | 30 + .../results/clientpositive/udf_reverse.q.out | 30 + .../test/results/clientpositive/union10.q.out | 56 + .../test/results/clientpositive/union12.q.out | 56 + .../test/results/clientpositive/union17.q.out | 96 +- .../test/results/clientpositive/union18.q.out | 140 +- .../test/results/clientpositive/union19.q.out | 109 +- .../test/results/clientpositive/union22.q.out | 89 +- .../test/results/clientpositive/union28.q.out | 56 + .../test/results/clientpositive/union29.q.out | 56 + .../test/results/clientpositive/union30.q.out | 69 + .../test/results/clientpositive/union31.q.out | 290 ++- .../test/results/clientpositive/union33.q.out | 86 + ql/src/test/results/clientpositive/union4.q.out | 43 + ql/src/test/results/clientpositive/union6.q.out | 43 + .../clientpositive/union_lateralview.q.out | 44 +- .../results/clientpositive/union_stats.q.out | 4 +- .../clientpositive/vector_groupby4.q.out | 104 +- .../clientpositive/vector_groupby6.q.out | 104 +- .../clientpositive/vectorized_context.q.out | 32 +- 637 files changed, 46448 insertions(+), 12315 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out ---------------------------------------------------------------------- diff --git a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out index f14c3d6..0efe121 100644 --- a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out +++ b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out @@ -35,12 +35,12 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 + Stage-5 depends on stages: Stage-3, Stage-2, Stage-1 Stage-1 is a root stage Stage-4 is a root stage Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8 Stage-7 Stage-3 depends on stages: Stage-7, Stage-6, Stage-9 - Stage-5 depends on stages: Stage-3 Stage-6 Stage-8 Stage-9 depends on stages: Stage-8 @@ -58,6 +58,14 @@ STAGE PLANS: Insert operator: Insert + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.src_x1 + Stage: Stage-1 Pre Insert operator: Pre-Insert task @@ -83,6 +91,19 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_x1 + Select Operator + expressions: _col0 (type: string), '' (type: string) + outputColumnNames: key, value + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>) Filter Operator predicate: ((key < 100) and (key > 50)) (type: boolean) Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE @@ -98,6 +119,19 @@ STAGE PLANS: output format: org.apache.hadoop.hive.accumulo.mr.HiveAccumuloTableOutputFormat serde: org.apache.hadoop.hive.accumulo.serde.AccumuloSerDe name: default.src_x2 + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-10 Conditional Operator @@ -118,10 +152,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_x1 - Stage: Stage-5 - Stats Work - Basic Stats Work: - Stage: Stage-6 Map Reduce Map Operator Tree: http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index a0b163d..dc31505 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1738,7 +1738,7 @@ public class HiveConf extends Configuration { "This many percentage of rows will be estimated as number of nulls in absence of statistics."), HIVESTATSAUTOGATHER("hive.stats.autogather", true, "A flag to gather statistics (only basic) automatically during the INSERT OVERWRITE command."), - HIVESTATSCOLAUTOGATHER("hive.stats.column.autogather", false, + HIVESTATSCOLAUTOGATHER("hive.stats.column.autogather", true, "A flag to gather column statistics automatically."), HIVESTATSDBCLASS("hive.stats.dbclass", "fs", new PatternSet("custom", "fs"), "The storage that stores temporary Hive statistics. In filesystem based statistics collection ('fs'), \n" + http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/contrib/src/test/results/clientpositive/serde_typedbytes.q.out ---------------------------------------------------------------------- diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes.q.out index c844a70..c9d0fb6 100644 --- a/contrib/src/test/results/clientpositive/serde_typedbytes.q.out +++ b/contrib/src/test/results/clientpositive/serde_typedbytes.q.out @@ -72,6 +72,32 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-7 Conditional Operator @@ -95,6 +121,10 @@ STAGE PLANS: Stage: Stage-2 Stats Work Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.dest1 Stage: Stage-3 Map Reduce http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out ---------------------------------------------------------------------- diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out index c0228aa..715dc95 100644 --- a/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out +++ b/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out @@ -68,6 +68,32 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-7 Conditional Operator @@ -91,6 +117,10 @@ STAGE PLANS: Stage: Stage-2 Stats Work Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: smallint, string + Table: default.dest1 Stage: Stage-3 Map Reduce http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out ---------------------------------------------------------------------- diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out index 0b7541a..b66d504 100644 --- a/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out +++ b/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out @@ -68,6 +68,32 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-7 Conditional Operator @@ -91,6 +117,10 @@ STAGE PLANS: Stage: Stage-2 Stats Work Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.dest1 Stage: Stage-3 Map Reduce http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out ---------------------------------------------------------------------- diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out index 981ff21..45a2092 100644 --- a/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out +++ b/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out @@ -77,6 +77,26 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col1 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Move Operator @@ -91,6 +111,10 @@ STAGE PLANS: Stage: Stage-2 Stats Work Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.dest1 PREHOOK: query: FROM ( FROM src http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out ---------------------------------------------------------------------- diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out index 5a7df3c..4beb35a 100644 --- a/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out +++ b/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out @@ -72,6 +72,32 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-7 Conditional Operator @@ -95,6 +121,10 @@ STAGE PLANS: Stage: Stage-2 Stats Work Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.dest1 Stage: Stage-3 Map Reduce http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/data/conf/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml index 6908bb1..9f6aec1 100644 --- a/data/conf/hive-site.xml +++ b/data/conf/hive-site.xml @@ -302,12 +302,15 @@ <value>true</value> </property> - <property> <name>hive.llap.io.allocator.direct</name> <value>false</value> </property> +<property> + <name>hive.stats.column.autogather</name> + <value>true</value> +</property> <property> <name>hive.materializedview.rewriting</name> http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/data/conf/spark/local/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/spark/local/hive-site.xml b/data/conf/spark/local/hive-site.xml index 8bade0f..fd0e6a0 100644 --- a/data/conf/spark/local/hive-site.xml +++ b/data/conf/spark/local/hive-site.xml @@ -255,4 +255,10 @@ <description>Internal marker for test. Used for masking env-dependent values</description> </property> +<!-- remove this after HIVE-18139 --> +<property> + <name>hive.stats.column.autogather</name> + <value>false</value> +</property> + </configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/data/conf/spark/standalone/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml index 90112e9..1e5bd65 100644 --- a/data/conf/spark/standalone/hive-site.xml +++ b/data/conf/spark/standalone/hive-site.xml @@ -255,4 +255,10 @@ <description>Internal marker for test. Used for masking env-dependent values</description> </property> +<!-- remove this after HIVE-18139 --> +<property> + <name>hive.stats.column.autogather</name> + <value>false</value> +</property> + </configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/data/conf/spark/yarn-client/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml index 64b56a5..a9a788b 100644 --- a/data/conf/spark/yarn-client/hive-site.xml +++ b/data/conf/spark/yarn-client/hive-site.xml @@ -300,4 +300,10 @@ <value>99</value> </property> +<!-- remove this after HIVE-18139 --> +<property> + <name>hive.stats.column.autogather</name> + <value>false</value> +</property> + </configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out b/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out index efc4a11..7e47a65 100644 --- a/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out +++ b/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out @@ -35,12 +35,12 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 + Stage-5 depends on stages: Stage-3, Stage-2, Stage-1 Stage-1 is a root stage Stage-4 is a root stage Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8 Stage-7 Stage-3 depends on stages: Stage-7, Stage-6, Stage-9 - Stage-5 depends on stages: Stage-3 Stage-6 Stage-8 Stage-9 depends on stages: Stage-8 @@ -58,6 +58,14 @@ STAGE PLANS: Insert operator: Insert + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.src_x1 + Stage: Stage-1 Pre Insert operator: Pre-Insert task @@ -83,6 +91,19 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_x1 + Select Operator + expressions: _col0 (type: string), '' (type: string) + outputColumnNames: key, value + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>) Filter Operator predicate: ((key < 100) and (key > 50)) (type: boolean) Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE @@ -98,6 +119,19 @@ STAGE PLANS: output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat serde: org.apache.hadoop.hive.hbase.HBaseSerDe name: default.src_x2 + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-10 Conditional Operator @@ -118,10 +152,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_x1 - Stage: Stage-5 - Stats Work - Basic Stats Work: - Stage: Stage-6 Map Reduce Map Operator Tree: http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out b/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out index cae2a13..5d95dbd 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out @@ -76,6 +76,26 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.blobstore_table + Select Operator + expressions: _col0 (type: int) + outputColumnNames: cnt + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(cnt, 'hll') + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Move Operator @@ -90,6 +110,10 @@ STAGE PLANS: Stage: Stage-2 Stats Work Basic Stats Work: + Column Stats Desc: + Columns: cnt + Column Types: int + Table: default.blobstore_table PREHOOK: query: SELECT * FROM blobstore_table PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out index daf95c3..bbd81d1 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out @@ -211,6 +211,11 @@ STAGE PLANS: Stats Work Basic Stats Work: Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Column Stats Desc: + Columns: id + Column Types: int + Table: default.table1 + Is Table Level Stats: false PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out index 5349210..315aedb 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out @@ -73,7 +73,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -95,6 +95,22 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false + Select Operator + expressions: _col0 (type: int) + outputColumnNames: id + Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(id, 'hll') + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -134,6 +150,35 @@ STAGE PLANS: name: default.values__tmp__table__3 Truncated Path -> Alias: #### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary> + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-7 Conditional Operator @@ -147,7 +192,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -171,6 +216,11 @@ STAGE PLANS: Stats Work Basic Stats Work: Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Column Stats Desc: + Columns: id + Column Types: int + Table: default.table1 + Is Table Level Stats: true Stage: Stage-3 Map Reduce @@ -186,7 +236,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -217,7 +267,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -239,7 +289,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -271,7 +321,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -305,7 +355,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -336,7 +386,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -358,7 +408,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out index 6e95fd1..17db9db 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out @@ -107,7 +107,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true","key":"true"}} bucket_count -1 column.name.delimiter , columns id,key @@ -129,7 +129,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true","key":"true"}} bucket_count -1 column.name.delimiter , columns id,key @@ -358,7 +358,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true","key":"true"}} bucket_count -1 column.name.delimiter , columns id,key @@ -380,7 +380,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true","key":"true"}} bucket_count -1 column.name.delimiter , columns id,key http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out index 42b9821..2192e15 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out @@ -229,6 +229,11 @@ STAGE PLANS: Stats Work Basic Stats Work: Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Column Stats Desc: + Columns: id + Column Types: int + Table: default.table1 + Is Table Level Stats: false PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE http://git-wip-us.apache.org/repos/asf/hive/blob/3bbc24d2/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out index cae1a5b..10911a5 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out @@ -81,7 +81,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -103,6 +103,22 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false + Select Operator + expressions: _col0 (type: int) + outputColumnNames: id + Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(id, 'hll') + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -142,6 +158,35 @@ STAGE PLANS: name: default.values__tmp__table__3 Truncated Path -> Alias: #### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary> + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-7 Conditional Operator @@ -155,7 +200,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -179,6 +224,11 @@ STAGE PLANS: Stats Work Basic Stats Work: Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Column Stats Desc: + Columns: id + Column Types: int + Table: default.table1 + Is Table Level Stats: true Stage: Stage-3 Map Reduce @@ -194,7 +244,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -225,7 +275,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -247,7 +297,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -279,7 +329,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -313,7 +363,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -344,7 +394,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id @@ -366,7 +416,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}} bucket_count -1 column.name.delimiter , columns id
