HIVE-18908: FULL OUTER JOIN to MapJoin (Matt McCline, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a37827ec Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a37827ec Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a37827ec Branch: refs/heads/master Commit: a37827ecd557c7f7d69f3b2ccdbf6535908b1461 Parents: 45163ee Author: Matt McCline <mmccl...@hortonworks.com> Authored: Sun Sep 16 09:04:59 2018 -0500 Committer: Matt McCline <mmccl...@hortonworks.com> Committed: Sun Sep 16 09:04:59 2018 -0500 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 10 +- data/files/fullouter_long_big_1a.txt | 11 + data/files/fullouter_long_big_1a_nonull.txt | 10 + data/files/fullouter_long_big_1b.txt | 13 + data/files/fullouter_long_big_1c.txt | 11 + data/files/fullouter_long_big_1d.txt | 12 + data/files/fullouter_long_small_1a.txt | 54 + data/files/fullouter_long_small_1a_nonull.txt | 51 + data/files/fullouter_long_small_1b.txt | 72 + data/files/fullouter_long_small_1c.txt | 81 + data/files/fullouter_long_small_1d.txt | 39 + data/files/fullouter_multikey_big_1a.txt | 13 + data/files/fullouter_multikey_big_1a_nonull.txt | 10 + data/files/fullouter_multikey_big_1b.txt | 17 + data/files/fullouter_multikey_small_1a.txt | 92 + .../fullouter_multikey_small_1a_nonull.txt | 90 + data/files/fullouter_multikey_small_1b.txt | 118 + data/files/fullouter_string_big_1a.txt | 13 + data/files/fullouter_string_big_1a_nonull.txt | 12 + data/files/fullouter_string_big_1a_old.txt | 13 + data/files/fullouter_string_small_1a.txt | 38 + data/files/fullouter_string_small_1a_nonull.txt | 35 + data/files/fullouter_string_small_1a_old.txt | 38 + .../vectorization/mapjoin/AbstractMapJoin.java | 66 +- .../mapjoin/MapJoinMultiKeyBenchBase.java | 3 +- .../mapjoin/MapJoinOneLongKeyBenchBase.java | 3 +- .../mapjoin/MapJoinOneStringKeyBenchBase.java | 3 +- .../test/resources/testconfiguration.properties | 5 + .../hadoop/hive/ql/exec/CommonJoinOperator.java | 11 +- .../apache/hadoop/hive/ql/exec/ExplainTask.java | 187 +- .../apache/hadoop/hive/ql/exec/JoinUtil.java | 2 +- .../hadoop/hive/ql/exec/MapJoinOperator.java | 199 +- .../apache/hadoop/hive/ql/exec/Operator.java | 86 +- .../hadoop/hive/ql/exec/TableScanOperator.java | 6 +- .../persistence/BytesBytesMultiHashMap.java | 71 +- .../ql/exec/persistence/HashMapWrapper.java | 22 + .../persistence/HybridHashTableContainer.java | 118 +- .../persistence/MapJoinBytesTableContainer.java | 199 +- .../hive/ql/exec/persistence/MapJoinKey.java | 15 + .../persistence/MapJoinObjectSerDeContext.java | 17 + .../exec/persistence/MapJoinTableContainer.java | 61 +- .../hive/ql/exec/persistence/MatchTracker.java | 154 + .../ReusableGetAdaptorDirectAccess.java | 4 +- .../ql/exec/persistence/UnwrapRowContainer.java | 12 +- .../hive/ql/exec/tez/ReduceRecordProcessor.java | 2 +- .../vector/VectorAppMasterEventOperator.java | 2 +- .../hive/ql/exec/vector/VectorAssignRow.java | 12 + .../hive/ql/exec/vector/VectorCopyRow.java | 21 +- .../ql/exec/vector/VectorDeserializeRow.java | 32 + .../ql/exec/vector/VectorFilterOperator.java | 2 +- .../ql/exec/vector/VectorGroupByOperator.java | 2 +- .../ql/exec/vector/VectorLimitOperator.java | 2 +- .../exec/vector/VectorMapJoinBaseOperator.java | 23 +- .../ql/exec/vector/VectorMapJoinOperator.java | 23 +- .../exec/vector/VectorSMBMapJoinOperator.java | 2 +- .../ql/exec/vector/VectorSelectOperator.java | 4 +- .../ql/exec/vector/VectorTopNKeyOperator.java | 2 +- .../mapjoin/VectorMapJoinCommonOperator.java | 409 +- .../VectorMapJoinFullOuterLongOperator.java | 68 + .../VectorMapJoinFullOuterMultiKeyOperator.java | 71 + .../VectorMapJoinFullOuterStringOperator.java | 71 + .../VectorMapJoinGenerateResultOperator.java | 163 +- ...pJoinInnerBigOnlyGenerateResultOperator.java | 18 +- .../VectorMapJoinInnerBigOnlyLongOperator.java | 63 +- ...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 60 +- ...VectorMapJoinInnerBigOnlyStringOperator.java | 50 +- ...ectorMapJoinInnerGenerateResultOperator.java | 22 +- .../mapjoin/VectorMapJoinInnerLongOperator.java | 64 +- .../VectorMapJoinInnerMultiKeyOperator.java | 58 +- .../VectorMapJoinInnerStringOperator.java | 50 +- ...orMapJoinLeftSemiGenerateResultOperator.java | 12 +- .../VectorMapJoinLeftSemiLongOperator.java | 64 +- .../VectorMapJoinLeftSemiMultiKeyOperator.java | 58 +- .../VectorMapJoinLeftSemiStringOperator.java | 50 +- ...ectorMapJoinOuterGenerateResultOperator.java | 382 +- .../mapjoin/VectorMapJoinOuterLongOperator.java | 117 +- .../VectorMapJoinOuterMultiKeyOperator.java | 110 +- .../VectorMapJoinOuterStringOperator.java | 99 +- .../fast/VectorMapJoinFastBytesHashMap.java | 162 +- .../VectorMapJoinFastBytesHashMapStore.java | 30 +- .../VectorMapJoinFastBytesHashMultiSet.java | 5 +- .../fast/VectorMapJoinFastBytesHashSet.java | 5 +- .../fast/VectorMapJoinFastBytesHashTable.java | 7 +- .../mapjoin/fast/VectorMapJoinFastHashMap.java | 8 +- .../fast/VectorMapJoinFastHashMultiSet.java | 8 +- .../mapjoin/fast/VectorMapJoinFastHashSet.java | 6 +- .../fast/VectorMapJoinFastHashTable.java | 33 +- .../mapjoin/fast/VectorMapJoinFastKeyStore.java | 34 +- .../fast/VectorMapJoinFastLongHashMap.java | 166 +- .../fast/VectorMapJoinFastLongHashMultiSet.java | 38 +- .../fast/VectorMapJoinFastLongHashSet.java | 27 +- .../fast/VectorMapJoinFastLongHashTable.java | 26 +- .../fast/VectorMapJoinFastMultiKeyHashMap.java | 8 +- .../VectorMapJoinFastMultiKeyHashMultiSet.java | 8 +- .../fast/VectorMapJoinFastMultiKeyHashSet.java | 8 +- .../VectorMapJoinFastNonMatchedIterator.java | 40 + .../fast/VectorMapJoinFastStringCommon.java | 12 +- .../fast/VectorMapJoinFastStringHashMap.java | 16 +- .../VectorMapJoinFastStringHashMultiSet.java | 21 +- .../fast/VectorMapJoinFastStringHashSet.java | 13 +- .../fast/VectorMapJoinFastTableContainer.java | 64 +- .../hashtable/VectorMapJoinBytesHashMap.java | 16 + .../mapjoin/hashtable/VectorMapJoinHashMap.java | 1 - .../hashtable/VectorMapJoinHashTable.java | 8 +- .../hashtable/VectorMapJoinLongHashMap.java | 13 + .../hashtable/VectorMapJoinLongHashTable.java | 1 - .../VectorMapJoinNonMatchedIterator.java | 72 + .../VectorMapJoinOptimizedCreateHashTable.java | 12 - .../VectorMapJoinOptimizedHashMap.java | 80 +- .../VectorMapJoinOptimizedHashMultiSet.java | 2 +- .../VectorMapJoinOptimizedHashSet.java | 2 +- .../VectorMapJoinOptimizedHashTable.java | 27 +- .../VectorMapJoinOptimizedLongCommon.java | 2 - .../VectorMapJoinOptimizedLongHashMap.java | 122 +- .../VectorMapJoinOptimizedMultiKeyHashMap.java | 2 - ...ectorMapJoinOptimizedNonMatchedIterator.java | 55 + .../VectorMapJoinOptimizedStringCommon.java | 19 +- .../VectorMapJoinOptimizedStringHashMap.java | 76 +- .../hive/ql/optimizer/AbstractSMBJoinProc.java | 3 + .../hive/ql/optimizer/ConvertJoinMapJoin.java | 175 +- .../optimizer/FullOuterMapJoinOptimization.java | 95 + .../hive/ql/optimizer/MapJoinProcessor.java | 260 +- .../ql/optimizer/SparkMapJoinProcessor.java | 3 + .../hive/ql/optimizer/physical/Vectorizer.java | 206 +- .../optimizer/spark/SparkMapJoinOptimizer.java | 3 + .../hive/ql/parse/ExplainConfiguration.java | 9 + .../hive/ql/parse/ExplainSemanticAnalyzer.java | 2 + .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 1 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 2 + .../hadoop/hive/ql/parse/IdentifiersParser.g | 1 + .../org/apache/hadoop/hive/ql/plan/Explain.java | 2 +- .../apache/hadoop/hive/ql/plan/ExplainWork.java | 4 + .../hadoop/hive/ql/plan/JoinCondDesc.java | 2 +- .../apache/hadoop/hive/ql/plan/JoinDesc.java | 2 +- .../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 210 +- .../hadoop/hive/ql/plan/MergeJoinWork.java | 57 + .../ql/plan/OperatorExplainVectorization.java | 19 + .../hadoop/hive/ql/plan/ReduceSinkDesc.java | 74 +- .../hadoop/hive/ql/plan/VectorMapJoinDesc.java | 16 +- .../hadoop/hive/ql/plan/VectorMapJoinInfo.java | 97 +- .../persistence/TestBytesBytesMultiHashMap.java | 8 +- .../CollectorTestOperator.java | 22 + .../RowCollectorTestOperator.java | 22 +- .../RowVectorCollectorTestOperator.java | 10 + .../util/rowobjects/RowTestObjectsMultiSet.java | 169 +- .../TestTimestampWritableAndColumnVector.java | 2 - .../exec/vector/TestVectorSelectOperator.java | 3 +- .../ql/exec/vector/VectorRandomRowSource.java | 45 +- .../exec/vector/mapjoin/MapJoinTestConfig.java | 734 +- .../ql/exec/vector/mapjoin/MapJoinTestData.java | 269 +- .../vector/mapjoin/MapJoinTestDescription.java | 249 +- .../ql/exec/vector/mapjoin/NoOpExpression.java | 52 + .../vector/mapjoin/TestMapJoinOperator.java | 1924 +++- .../vector/mapjoin/fast/CheckFastHashTable.java | 184 +- ...VectorMapJoinFastBytesHashMapNonMatched.java | 167 + .../fast/TestVectorMapJoinFastBytesHashSet.java | 16 +- ...tVectorMapJoinFastLongHashMapNonMatched.java | 196 + .../exec/vector/mapjoin/fast/VerifyFastRow.java | 2 - .../util/batchgen/VectorBatchGenerator.java | 29 +- .../batchgen/VectorColumnGroupGenerator.java | 290 +- .../ql/plan/mapping/TestCounterMapping.java | 2 +- ql/src/test/queries/clientnegative/join32.q | 3 + .../queries/clientpositive/auto_join_filters.q | 29 + .../queries/clientpositive/auto_join_nulls.q | 9 + .../clientpositive/bucket_map_join_tez1.q | 2 +- .../clientpositive/correlationoptimizer1.q | 30 + .../clientpositive/correlationoptimizer2.q | 107 + .../clientpositive/correlationoptimizer4.q | 47 +- .../fullouter_mapjoin_1_optimized.q | 290 + .../queries/clientpositive/join32_lessSize.q | 1 + ql/src/test/queries/clientpositive/join33.q | 1 + ql/src/test/queries/clientpositive/mapjoin2.q | 14 + ql/src/test/queries/clientpositive/mapjoin46.q | 109 + .../clientpositive/parquet_vectorization_0.q | 14 +- .../clientpositive/parquet_vectorization_1.q | 2 +- .../clientpositive/parquet_vectorization_10.q | 2 +- .../clientpositive/parquet_vectorization_11.q | 2 +- .../clientpositive/parquet_vectorization_12.q | 2 +- .../clientpositive/parquet_vectorization_13.q | 2 +- .../clientpositive/parquet_vectorization_14.q | 2 +- .../clientpositive/parquet_vectorization_15.q | 2 +- .../clientpositive/parquet_vectorization_16.q | 2 +- .../clientpositive/parquet_vectorization_17.q | 2 +- .../clientpositive/parquet_vectorization_2.q | 2 +- .../clientpositive/parquet_vectorization_3.q | 2 +- .../clientpositive/parquet_vectorization_4.q | 2 +- .../clientpositive/parquet_vectorization_5.q | 2 +- .../clientpositive/parquet_vectorization_6.q | 2 +- .../clientpositive/parquet_vectorization_7.q | 2 +- .../clientpositive/parquet_vectorization_8.q | 2 +- .../clientpositive/parquet_vectorization_9.q | 2 +- .../parquet_vectorization_limit.q | 12 +- .../clientpositive/tez_dynpart_hashjoin_3.q | 6 + ql/src/test/queries/clientpositive/union14.q | 4 +- ql/src/test/queries/clientpositive/union7.q | 3 +- ql/src/test/queries/clientpositive/union_null.q | 3 +- ql/src/test/queries/clientpositive/union_view.q | 2 + .../clientpositive/vector_full_outer_join.q | 82 + .../vector_fullouter_mapjoin_1_fast.q | 290 + .../vector_fullouter_mapjoin_1_optimized.q | 290 + ...tor_fullouter_mapjoin_1_optimized_passthru.q | 290 + .../test/queries/clientpositive/vector_join30.q | 219 +- .../clientpositive/vector_join_filters.q | 14 + .../queries/clientpositive/vector_join_nulls.q | 16 + .../clientpositive/vector_left_outer_join2.q | 13 +- .../clientpositive/vector_leftsemi_mapjoin.q | 290 +- .../clientpositive/vector_nullsafe_join.q | 8 + .../queries/clientpositive/vectorized_join46.q | 35 +- .../clientpositive/vectorized_join46_mr.q | 228 + .../clientpositive/annotate_stats_join.q.out | 2 +- .../results/clientpositive/auto_join18.q.out | 2 +- .../auto_join18_multi_distinct.q.out | 2 +- .../results/clientpositive/auto_join6.q.out | 2 +- .../results/clientpositive/auto_join7.q.out | 2 +- .../clientpositive/beeline/mapjoin2.q.out | 532 + .../clientpositive/beeline/smb_mapjoin_1.q.out | 4 +- .../clientpositive/beeline/smb_mapjoin_2.q.out | 4 +- .../clientpositive/beeline/smb_mapjoin_3.q.out | 4 +- .../clientpositive/beeline/smb_mapjoin_7.q.out | 2 +- .../results/clientpositive/cbo_rp_join1.q.out | 8 +- .../clientpositive/correlationoptimizer8.q.out | 2 +- .../clientpositive/fullouter_mapjoin_1.q.out | 176 + .../clientpositive/infer_join_preds.q.out | 2 +- ql/src/test/results/clientpositive/join18.q.out | 2 +- .../clientpositive/join18_multi_distinct.q.out | 2 +- ql/src/test/results/clientpositive/join45.q.out | 4 +- ql/src/test/results/clientpositive/join46.q.out | 10 +- ql/src/test/results/clientpositive/join47.q.out | 4 +- ql/src/test/results/clientpositive/join6.q.out | 2 +- ql/src/test/results/clientpositive/join7.q.out | 2 +- .../clientpositive/join_filters_overlap.q.out | 2 +- .../clientpositive/llap/acid_no_buckets.q.out | 62 +- .../clientpositive/llap/auto_join_filters.q.out | 389 + .../clientpositive/llap/auto_join_nulls.q.out | 133 + .../llap/convert_decimal64_to_decimal.q.out | 14 +- .../llap/correlationoptimizer1.q.out | 288 +- .../llap/correlationoptimizer2.q.out | 1346 ++- .../llap/correlationoptimizer4.q.out | 921 +- .../clientpositive/llap/cross_prod_1.q.out | 2 +- .../clientpositive/llap/explainuser_4.q.out | 6 +- .../llap/fullouter_mapjoin_1_optimized.q.out | 3139 ++++++ .../results/clientpositive/llap/join46.q.out | 10 +- .../llap/join_max_hashtable.q.out | 4 + .../results/clientpositive/llap/llap_acid.q.out | 14 +- .../clientpositive/llap/llap_acid_fast.q.out | 14 +- .../clientpositive/llap/llap_partitioned.q.out | 16 +- .../llap/llap_vector_nohybridgrace.q.out | 2 + .../results/clientpositive/llap/mapjoin2.q.out | 546 + .../results/clientpositive/llap/mapjoin46.q.out | 956 +- .../results/clientpositive/llap/mergejoin.q.out | 138 +- .../clientpositive/llap/orc_llap_counters.q.out | 176 +- .../llap/orc_llap_counters1.q.out | 12 +- .../clientpositive/llap/partialdhj.q.out | 4 + .../results/clientpositive/llap/reopt_dpp.q.out | 2 +- .../clientpositive/llap/reopt_semijoin.q.out | 2 +- .../clientpositive/llap/results_cache_2.q.out | 6 +- .../clientpositive/llap/runtime_stats_hs2.q.out | 12 +- .../results/clientpositive/llap/semijoin.q.out | 4 +- .../clientpositive/llap/smb_mapjoin_4.q.out | 12 +- .../clientpositive/llap/smb_mapjoin_5.q.out | 12 +- .../llap/tez_dynpart_hashjoin_1.q.out | 4 + .../llap/tez_dynpart_hashjoin_3.q.out | 109 + .../llap/tez_input_counters.q.out | 32 +- .../llap/tez_vector_dynpart_hashjoin_1.q.out | 3 + .../results/clientpositive/llap/union7.q.out | 16 +- .../llap/vector_aggregate_9.q.out | 9 +- .../llap/vector_aggregate_without_gby.q.out | 3 +- .../llap/vector_auto_smb_mapjoin_14.q.out | 45 + .../llap/vector_between_columns.q.out | 6 + .../llap/vector_binary_join_groupby.q.out | 2 + .../llap/vector_case_when_2.q.out | 8 +- .../llap/vector_char_mapjoin1.q.out | 3 + .../clientpositive/llap/vector_coalesce_3.q.out | 15 +- .../clientpositive/llap/vector_coalesce_4.q.out | 4 +- .../llap/vector_complex_all.q.out | 18 +- .../llap/vector_complex_join.q.out | 3 + .../clientpositive/llap/vector_date_1.q.out | 20 +- .../clientpositive/llap/vector_decimal_1.q.out | 27 +- .../llap/vector_decimal_10_0.q.out | 6 +- .../clientpositive/llap/vector_decimal_2.q.out | 48 +- .../clientpositive/llap/vector_decimal_6.q.out | 16 +- .../llap/vector_decimal_aggregate.q.out | 16 +- .../llap/vector_decimal_expressions.q.out | 6 +- .../llap/vector_decimal_mapjoin.q.out | 21 +- .../llap/vector_decimal_precision.q.out | 6 +- .../llap/vector_decimal_round.q.out | 24 +- .../llap/vector_decimal_round_2.q.out | 16 +- .../llap/vector_decimal_trailing.q.out | 4 +- .../llap/vector_decimal_udf.q.out | 50 +- .../llap/vector_full_outer_join.q.out | 1228 +++ .../llap/vector_fullouter_mapjoin_1_fast.q.out | 3945 +++++++ .../vector_fullouter_mapjoin_1_optimized.q.out | 3945 +++++++ ...fullouter_mapjoin_1_optimized_passthru.q.out | 3923 +++++++ .../llap/vector_groupby_cube1.q.out | 48 +- .../llap/vector_groupby_grouping_id1.q.out | 18 +- .../llap/vector_groupby_grouping_id2.q.out | 138 +- .../llap/vector_groupby_grouping_id3.q.out | 8 +- .../llap/vector_groupby_grouping_sets1.q.out | 26 +- .../llap/vector_groupby_grouping_sets2.q.out | 28 +- .../llap/vector_groupby_grouping_sets3.q.out | 4 +- .../vector_groupby_grouping_sets3_dec.q.out | 16 +- .../llap/vector_groupby_grouping_sets4.q.out | 61 +- .../llap/vector_groupby_grouping_sets5.q.out | 25 +- .../llap/vector_groupby_grouping_sets6.q.out | 6 +- .../vector_groupby_grouping_sets_grouping.q.out | 53 +- .../vector_groupby_grouping_sets_limit.q.out | 44 +- .../llap/vector_groupby_grouping_window.q.out | 10 +- .../llap/vector_groupby_mapjoin.q.out | 2 + .../llap/vector_groupby_rollup1.q.out | 40 +- .../llap/vector_groupby_sort_11.q.out | 21 +- .../llap/vector_groupby_sort_8.q.out | 3 +- .../llap/vector_include_no_sel.q.out | 3 + .../clientpositive/llap/vector_inner_join.q.out | 134 +- .../llap/vector_interval_mapjoin.q.out | 1 + .../clientpositive/llap/vector_join30.q.out | 2047 +++- .../llap/vector_join_filters.q.out | 868 +- .../clientpositive/llap/vector_join_nulls.q.out | 823 +- .../llap/vector_left_outer_join2.q.out | 120 +- .../llap/vector_leftsemi_mapjoin.q.out | 9583 +++++++++++------- .../clientpositive/llap/vector_like_2.q.out | 4 +- .../llap/vector_llap_io_data_conversion.q.out | 3 +- .../llap/vector_llap_text_1.q.out | 21 +- .../llap/vector_mapjoin_reduce.q.out | 4 + .../llap/vector_mr_diff_schema_alias.q.out | 6 + .../llap/vector_nullsafe_join.q.out | 298 +- .../llap/vector_orc_nested_column_pruning.q.out | 15 + .../clientpositive/llap/vector_order_null.q.out | 33 +- .../llap/vector_outer_join0.q.out | 34 +- .../llap/vector_outer_join1.q.out | 56 +- .../llap/vector_outer_join2.q.out | 27 +- .../llap/vector_outer_reference_windowed.q.out | 120 +- .../clientpositive/llap/vector_ptf_1.q.out | 6 +- .../llap/vector_ptf_part_simple.q.out | 156 +- .../vector_reduce_groupby_duplicate_cols.q.out | 10 +- .../llap/vector_reuse_scratchcols.q.out | 6 +- .../clientpositive/llap/vector_topnkey.q.out | 24 +- .../clientpositive/llap/vector_udf1.q.out | 6 +- .../clientpositive/llap/vector_windowing.q.out | 346 +- .../llap/vector_windowing_expressions.q.out | 55 +- .../llap/vector_windowing_gby.q.out | 17 +- .../llap/vector_windowing_gby2.q.out | 58 +- .../vector_windowing_multipartitioning.q.out | 60 +- .../llap/vector_windowing_navfn.q.out | 57 +- .../llap/vector_windowing_order_null.q.out | 45 +- .../vector_windowing_range_multiorder.q.out | 57 +- .../llap/vector_windowing_rank.q.out | 52 +- .../llap/vector_windowing_streaming.q.out | 5 +- .../llap/vector_windowing_windowspec.q.out | 60 +- .../llap/vector_windowing_windowspec4.q.out | 5 +- .../clientpositive/llap/vectorization_0.q.out | 42 +- .../clientpositive/llap/vectorization_1.q.out | 3 +- .../clientpositive/llap/vectorization_12.q.out | 8 +- .../clientpositive/llap/vectorization_13.q.out | 7 +- .../clientpositive/llap/vectorization_14.q.out | 8 +- .../clientpositive/llap/vectorization_15.q.out | 4 +- .../clientpositive/llap/vectorization_16.q.out | 4 +- .../clientpositive/llap/vectorization_17.q.out | 4 +- .../clientpositive/llap/vectorization_2.q.out | 3 +- .../clientpositive/llap/vectorization_3.q.out | 3 +- .../clientpositive/llap/vectorization_4.q.out | 3 +- .../clientpositive/llap/vectorization_5.q.out | 3 +- .../clientpositive/llap/vectorization_7.q.out | 3 +- .../clientpositive/llap/vectorization_8.q.out | 3 +- .../clientpositive/llap/vectorization_9.q.out | 4 +- .../llap/vectorization_limit.q.out | 30 +- .../llap/vectorization_nested_udf.q.out | 3 +- .../llap/vectorized_bucketmapjoin1.q.out | 9 + .../clientpositive/llap/vectorized_case.q.out | 6 +- .../llap/vectorized_distinct_gby.q.out | 9 +- .../vectorized_dynamic_partition_pruning.q.out | 87 + .../vectorized_dynamic_semijoin_reduction.q.out | 18 + ...vectorized_dynamic_semijoin_reduction2.q.out | 15 +- .../clientpositive/llap/vectorized_join46.q.out | 731 +- .../llap/vectorized_mapjoin.q.out | 1 + .../llap/vectorized_mapjoin3.q.out | 48 +- .../clientpositive/llap/vectorized_ptf.q.out | 136 +- .../llap/vectorized_shufflejoin.q.out | 3 + .../llap/vectorized_timestamp.q.out | 9 +- .../test/results/clientpositive/mapjoin2.q.out | 532 + .../test/results/clientpositive/mapjoin46.q.out | 907 +- .../test/results/clientpositive/mapjoin47.q.out | 4 +- .../test/results/clientpositive/mergejoin.q.out | 2 +- .../clientpositive/mergejoins_mixed.q.out | 8 +- .../results/clientpositive/optional_outer.q.out | 4 +- .../parquet_vectorization_0.q.out | 119 +- .../parquet_vectorization_1.q.out | 11 +- .../parquet_vectorization_10.q.out | 11 +- .../parquet_vectorization_11.q.out | 11 +- .../parquet_vectorization_12.q.out | 18 +- .../parquet_vectorization_13.q.out | 18 +- .../parquet_vectorization_14.q.out | 18 +- .../parquet_vectorization_15.q.out | 18 +- .../parquet_vectorization_16.q.out | 11 +- .../parquet_vectorization_17.q.out | 11 +- .../parquet_vectorization_2.q.out | 11 +- .../parquet_vectorization_3.q.out | 11 +- .../parquet_vectorization_4.q.out | 11 +- .../parquet_vectorization_5.q.out | 11 +- .../parquet_vectorization_6.q.out | 11 +- .../parquet_vectorization_7.q.out | 11 +- .../parquet_vectorization_8.q.out | 11 +- .../parquet_vectorization_9.q.out | 11 +- .../parquet_vectorization_limit.q.out | 59 +- .../clientpositive/perf/spark/query51.q.out | 2 +- .../clientpositive/perf/spark/query97.q.out | 2 +- .../clientpositive/skewjoin_mapjoin2.q.out | 4 +- .../results/clientpositive/skewjoinopt3.q.out | 4 +- .../results/clientpositive/smb_mapjoin_1.q.out | 4 +- .../results/clientpositive/smb_mapjoin_2.q.out | 4 +- .../results/clientpositive/smb_mapjoin_3.q.out | 4 +- .../results/clientpositive/smb_mapjoin_46.q.out | 8 +- .../results/clientpositive/smb_mapjoin_47.q.out | 4 +- .../results/clientpositive/smb_mapjoin_7.q.out | 2 +- .../spark/annotate_stats_join.q.out | 2 +- .../clientpositive/spark/auto_join14.q.out | 2 +- .../clientpositive/spark/auto_join18.q.out | 2 +- .../spark/auto_join18_multi_distinct.q.out | 2 +- .../clientpositive/spark/auto_join6.q.out | 2 +- .../clientpositive/spark/auto_join7.q.out | 2 +- .../spark/auto_join_filters.q.out | 377 + .../clientpositive/spark/auto_join_nulls.q.out | 129 + .../spark/bucket_map_join_tez1.q.out | 3506 +++---- .../results/clientpositive/spark/join18.q.out | 2 +- .../spark/join18_multi_distinct.q.out | 2 +- .../results/clientpositive/spark/join32.q.out | 166 +- .../results/clientpositive/spark/join33.q.out | 166 +- .../results/clientpositive/spark/join6.q.out | 2 +- .../results/clientpositive/spark/join7.q.out | 2 +- .../spark/join_filters_overlap.q.out | 2 +- .../clientpositive/spark/mergejoins_mixed.q.out | 8 +- .../spark/parquet_vectorization_0.q.out | 194 +- .../spark/parquet_vectorization_1.q.out | 20 +- .../spark/parquet_vectorization_10.q.out | 11 +- .../spark/parquet_vectorization_11.q.out | 11 +- .../spark/parquet_vectorization_12.q.out | 29 +- .../spark/parquet_vectorization_13.q.out | 29 +- .../spark/parquet_vectorization_14.q.out | 29 +- .../spark/parquet_vectorization_15.q.out | 13 +- .../spark/parquet_vectorization_16.q.out | 20 +- .../spark/parquet_vectorization_17.q.out | 20 +- .../spark/parquet_vectorization_2.q.out | 20 +- .../spark/parquet_vectorization_3.q.out | 20 +- .../spark/parquet_vectorization_4.q.out | 20 +- .../spark/parquet_vectorization_5.q.out | 20 +- .../spark/parquet_vectorization_6.q.out | 11 +- .../spark/parquet_vectorization_7.q.out | 20 +- .../spark/parquet_vectorization_8.q.out | 20 +- .../spark/parquet_vectorization_9.q.out | 20 +- .../spark/parquet_vectorization_limit.q.out | 115 +- .../results/clientpositive/spark/semijoin.q.out | 4 +- .../clientpositive/spark/skewjoinopt3.q.out | 4 +- .../clientpositive/spark/smb_mapjoin_1.q.out | 4 +- .../clientpositive/spark/smb_mapjoin_2.q.out | 4 +- .../clientpositive/spark/smb_mapjoin_3.q.out | 4 +- .../clientpositive/spark/smb_mapjoin_4.q.out | 12 +- .../clientpositive/spark/smb_mapjoin_5.q.out | 12 +- .../clientpositive/spark/smb_mapjoin_7.q.out | 2 +- ...k_vectorized_dynamic_partition_pruning.q.out | 389 +- .../results/clientpositive/spark/union14.q.out | 18 +- .../results/clientpositive/spark/union7.q.out | 18 +- .../clientpositive/spark/union_null.q.out | 10 +- .../clientpositive/spark/union_view.q.out | 4 +- .../spark/vector_decimal_aggregate.q.out | 16 +- .../spark/vector_inner_join.q.out | 102 +- .../spark/vector_mapjoin_reduce.q.out | 4 + .../spark/vector_outer_join0.q.out | 26 +- .../spark/vector_outer_join1.q.out | 43 +- .../spark/vector_outer_join2.q.out | 21 +- .../clientpositive/spark/vectorization_0.q.out | 42 +- .../clientpositive/spark/vectorization_1.q.out | 3 +- .../clientpositive/spark/vectorization_12.q.out | 8 +- .../clientpositive/spark/vectorization_13.q.out | 7 +- .../clientpositive/spark/vectorization_14.q.out | 8 +- .../clientpositive/spark/vectorization_15.q.out | 4 +- .../clientpositive/spark/vectorization_16.q.out | 4 +- .../clientpositive/spark/vectorization_17.q.out | 4 +- .../clientpositive/spark/vectorization_2.q.out | 3 +- .../clientpositive/spark/vectorization_3.q.out | 3 +- .../clientpositive/spark/vectorization_4.q.out | 3 +- .../clientpositive/spark/vectorization_5.q.out | 3 +- .../clientpositive/spark/vectorization_9.q.out | 4 +- .../spark/vectorization_nested_udf.q.out | 3 +- .../clientpositive/spark/vectorized_case.q.out | 6 +- .../spark/vectorized_mapjoin.q.out | 1 + .../clientpositive/spark/vectorized_ptf.q.out | 124 +- .../clientpositive/stat_estimate_drill.q.out | 40 +- .../stat_estimate_related_col.q.out | 38 +- .../clientpositive/tez/explainanalyze_3.q.out | 30 +- .../clientpositive/tez/explainanalyze_4.q.out | 6 +- .../results/clientpositive/union_null.q.out | 12 +- .../results/clientpositive/union_offcbo.q.out | 12 +- .../vector_left_outer_join2.q.out | 70 +- .../clientpositive/vectorized_join46_mr.q.out | 2050 ++++ .../apache/hadoop/hive/serde2/VerifyFast.java | 2 - .../exec/vector/TestTimestampColumnVector.java | 2 - 495 files changed, 51405 insertions(+), 11939 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0bfee2e..9d41790 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1679,7 +1679,15 @@ public class HiveConf extends Configuration { "joins unnecessary memory will be allocated and then trimmed."), HIVEHYBRIDGRACEHASHJOINBLOOMFILTER("hive.mapjoin.hybridgrace.bloomfilter", true, "Whether to " + "use BloomFilter in Hybrid grace hash join to minimize unnecessary spilling."), - + HIVEMAPJOINFULLOUTER("hive.mapjoin.full.outer", true, + "Whether to use MapJoin for FULL OUTER JOINs."), + HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE( + "hive.test.mapjoin.full.outer.override", + "none", new StringSet("none", "enable", "disable"), + "internal use only, used to override the hive.mapjoin.full.outer\n" + + "setting. Using enable will force it on and disable will force it off.\n" + + "The default none is do nothing, of course", + true), HIVESMBJOINCACHEROWS("hive.smbjoin.cache.rows", 10000, "How many rows with the same key value should be cached in memory per smb joined table."), HIVEGROUPBYMAPINTERVAL("hive.groupby.mapaggr.checkinterval", 100000, http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_big_1a.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_big_1a.txt b/data/files/fullouter_long_big_1a.txt new file mode 100644 index 0000000..8cf831f --- /dev/null +++ b/data/files/fullouter_long_big_1a.txt @@ -0,0 +1,11 @@ +-5310365297525168078 +-6187919478609154811 +968819023021777205 +3313583664488247651 +-5206670856103795573 +\N +-6187919478609154811 +1569543799237464101 +-6187919478609154811 +-8460550397108077433 +-6187919478609154811 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_big_1a_nonull.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_big_1a_nonull.txt b/data/files/fullouter_long_big_1a_nonull.txt new file mode 100644 index 0000000..b2325ad --- /dev/null +++ b/data/files/fullouter_long_big_1a_nonull.txt @@ -0,0 +1,10 @@ +1569543799237464101 +-6187919478609154811 +968819023021777205 +-8460550397108077433 +-6187919478609154811 +-5310365297525168078 +-6187919478609154811 +-5206670856103795573 +3313583664488247651 +-6187919478609154811 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_big_1b.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_big_1b.txt b/data/files/fullouter_long_big_1b.txt new file mode 100644 index 0000000..87c2b3c --- /dev/null +++ b/data/files/fullouter_long_big_1b.txt @@ -0,0 +1,13 @@ +\N +31713 +31713 +31713 +31713 +32030 +31713 +-25394 +31713 +31713 +31713 +31713 +31713 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_big_1c.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_big_1c.txt b/data/files/fullouter_long_big_1c.txt new file mode 100644 index 0000000..2d13c26 --- /dev/null +++ b/data/files/fullouter_long_big_1c.txt @@ -0,0 +1,11 @@ +1928928239,\N +-1437463633,YYXPPCH +-1437463633,TKTKGVGFW +1725068083,MKSCCE +1928928239,\N +\N,ABBZ +1928928239,AMKTIWQ +-1437463633,JU +1928928239,VAQHVRI +-1437463633,SOWDWMS +-1437463633,\N http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_big_1d.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_big_1d.txt b/data/files/fullouter_long_big_1d.txt new file mode 100644 index 0000000..4137f67 --- /dev/null +++ b/data/files/fullouter_long_big_1d.txt @@ -0,0 +1,12 @@ +-702028721 +-702028721 +-1780951928 +-670834064 +-814597051 +\N +-814597051 +-814597051 +-702028721 +-2038654700 +\N +-814597051 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_small_1a.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_small_1a.txt b/data/files/fullouter_long_small_1a.txt new file mode 100644 index 0000000..45d5825 --- /dev/null +++ b/data/files/fullouter_long_small_1a.txt @@ -0,0 +1,54 @@ +-1339636982994067311,2000-06-20 +-2575185053386712613,2105-01-21 +\N,2098-02-10 +-6784441713807772877,1845-02-16 +\N,2024-01-23 +-4224290881682877258,2185-07-08 +-614848861623872247,2101-05-25 +-2098090254092150988,2163-05-26 +434940853096155515,2275-02-08 +3873405809071478736,2034-06-09 +-2184423060953067642,1880-10-06 +7297177530102477725,1921-05-11 +7937120928560087303,2083-03-14 +\N,2242-02-08 +-2688622006344936758,2129-01-11 +214451696109242839,1977-01-04 +-4961171400048338491,2196-08-10 +4436884039838843341,2031-05-23 +2438535236662373438,1916-01-10 +6049335087268933751,2282-06-09 +8755921538765428593,1827-05-01 +5252407779338300447,2039-03-10 +-2184423060953067642,1853-07-06 +7297177530102477725,1926-04-12 +-2098090254092150988,1817-03-12 +-5754527700632192146,1958-07-15 +-614848861623872247,2112-11-09 +5246983111579595707,1817-07-01 +-2098090254092150988,2219-12-23 +-5706981533666803767,2151-06-09 +7297177530102477725,2125-08-26 +-7707546703881534780,2134-08-20 +214451696109242839,2179-04-18 +3845554233155411208,1805-11-10 +3905351789241845882,2045-12-05 +2438535236662373438,2026-06-23 +-2688622006344936758,1948-10-15 +6049335087268933751,2086-12-17 +-2575185053386712613,1809-07-12 +-327698348664467755,2222-10-15 +-4224290881682877258,1813-05-17 +3873405809071478736,2164-04-23 +-5706981533666803767,1800-09-20 +214451696109242839,1855-05-12 +2438535236662373438,1881-09-16 +5252407779338300447,2042-04-26 +-3655445881497026796,2108-08-16 +3905351789241845882,1866-07-28 +-6784441713807772877,2054-06-17 +5246983111579595707,2260-05-11 +-1339636982994067311,2008-12-03 +3873405809071478736,1918-11-20 +-4224290881682877258,2120-01-16 +3845554233155411208,2264-04-05 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_small_1a_nonull.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_small_1a_nonull.txt b/data/files/fullouter_long_small_1a_nonull.txt new file mode 100644 index 0000000..bf94d5a --- /dev/null +++ b/data/files/fullouter_long_small_1a_nonull.txt @@ -0,0 +1,51 @@ +5246983111579595707,1817-07-01 +4436884039838843341,2031-05-23 +-4224290881682877258,1813-05-17 +-4961171400048338491,2196-08-10 +-2575185053386712613,2105-01-21 +5252407779338300447,2042-04-26 +-614848861623872247,2101-05-25 +-2098090254092150988,2163-05-26 +2438535236662373438,1881-09-16 +214451696109242839,2179-04-18 +2438535236662373438,2026-06-23 +-2184423060953067642,1853-07-06 +3873405809071478736,2164-04-23 +214451696109242839,1855-05-12 +-6784441713807772877,1845-02-16 +-2688622006344936758,1948-10-15 +7297177530102477725,1921-05-11 +-2575185053386712613,1809-07-12 +3905351789241845882,2045-12-05 +3845554233155411208,1805-11-10 +-3655445881497026796,2108-08-16 +3905351789241845882,1866-07-28 +-1339636982994067311,2008-12-03 +7297177530102477725,2125-08-26 +7297177530102477725,1926-04-12 +-5706981533666803767,1800-09-20 +6049335087268933751,2282-06-09 +3845554233155411208,2264-04-05 +8755921538765428593,1827-05-01 +-1339636982994067311,2000-06-20 +-2098090254092150988,1817-03-12 +3873405809071478736,2034-06-09 +2438535236662373438,1916-01-10 +5246983111579595707,2260-05-11 +-5706981533666803767,2151-06-09 +-614848861623872247,2112-11-09 +-327698348664467755,2222-10-15 +-2184423060953067642,1880-10-06 +434940853096155515,2275-02-08 +-4224290881682877258,2120-01-16 +-5754527700632192146,1958-07-15 +-4224290881682877258,2185-07-08 +-2098090254092150988,2219-12-23 +-7707546703881534780,2134-08-20 +214451696109242839,1977-01-04 +-2688622006344936758,2129-01-11 +7937120928560087303,2083-03-14 +-6784441713807772877,2054-06-17 +3873405809071478736,1918-11-20 +6049335087268933751,2086-12-17 +5252407779338300447,2039-03-10 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_small_1b.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_small_1b.txt b/data/files/fullouter_long_small_1b.txt new file mode 100644 index 0000000..7d45fe4 --- /dev/null +++ b/data/files/fullouter_long_small_1b.txt @@ -0,0 +1,72 @@ +2748,2298-06-20 21:01:24 +11232,2533-11-26 12:22:18 +\N,2124-05-07 15:01:19.021 +3198,2428-06-13 16:21:33.955 +-7624,2219-12-03 17:07:19 +24870,2752-12-26 12:32:23.03685163 +14865,2943-03-21 00:42:10.505 +-8624,2644-05-04 04:45:07.839 +-30059,2269-05-04 21:23:44.000339209 +14865,2079-10-06 16:54:35.117 +-8435,2834-12-06 16:38:18.901 +10553,2168-05-05 21:10:59.000152113 +-8624,2282-03-28 07:58:16 +-15361,2219-09-15 20:15:03.000169887 +-14172,1918-09-13 11:44:24.496926711 +26484,1919-03-04 07:32:37.519 +-14172,2355-01-14 23:23:34 +-24775,2920-08-06 15:58:28.261059449 +-23117,2037-01-05 21:52:30.685952759 +17125,2236-07-14 01:54:40.927230276 +21181,2253-03-12 11:55:48.332 +-7373,2662-10-28 12:07:02.000526564 +-8087,2550-06-26 23:57:42.588007617 +29407,2385-12-14 06:03:39.597 +21181,2434-02-20 00:46:29.633 +-14172,2809-06-07 02:10:58 +13598,2421-05-20 14:18:31.000264698 +2748,2759-02-13 18:04:36.000307355 +-22422,1949-03-13 00:07:53.075 +26484,2953-03-10 02:05:26.508953676 +4510,2777-03-24 03:44:28.000169723 +-24775,2035-03-26 08:11:23.375224153 +-30059,2713-10-13 09:28:49 +-20517,2774-06-23 12:04:06.5 +11232,2038-04-06 14:53:59 +32030,2101-09-09 07:35:05.145 +-29600,2333-11-02 15:06:30 +-30306,2619-05-24 10:35:58.000774018 +-7624,2289-08-28 00:14:34 +-4279,2470-08-12 11:21:14.000955747 +-4279,2214-09-10 03:53:06 +-26998,2428-12-26 07:53:45.96925825 +17125,2629-11-15 15:34:52 +-8087,2923-07-02 11:40:26.115 +2632,2561-12-15 15:42:27 +21436,2696-05-08 05:19:24.112 +\N,2971-08-07 12:02:11.000948152 +-7624,2623-03-20 03:18:45.00006465 +-26998,2926-07-18 09:02:46.077 +11232,2507-01-27 22:04:22.49661421 +-30059,2420-12-10 22:12:30 +-15427,2355-01-08 12:34:11.617 +3198,2223-04-14 13:20:49 +-19167,2319-08-26 11:07:11.268 +14865,2220-02-28 03:41:36 +-20517,2233-12-20 04:06:56.666522799 +-15427,2046-06-07 22:58:40.728 +2748,2862-04-20 13:12:39.482805897 +-8435,2642-02-07 11:45:04.353231638 +-19167,2230-12-22 20:25:39.000242111 +-15427,2023-11-09 19:31:21 +13598,2909-06-25 23:22:50 +21436,2526-09-22 23:44:55 +-15361,2434-08-13 20:37:07.000172979 +4510,2293-01-17 13:47:41.00001006 +-8624,2120-02-15 15:36:40.000758423 +-22422,2337-07-19 06:33:02.000353352 +-26998,2268-08-04 12:48:11.848006292 +-22422,2982-12-28 06:30:26.000883228 +\N,2933-06-20 11:48:09.000839488 +3198,2736-12-20 03:59:50.343550301 +-20824,2478-11-05 00:28:05 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_small_1c.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_small_1c.txt b/data/files/fullouter_long_small_1c.txt new file mode 100644 index 0000000..ff323d3 --- /dev/null +++ b/data/files/fullouter_long_small_1c.txt @@ -0,0 +1,81 @@ +-1093006502,-69.55665828 +452719211,83003.43722 +1242586043,71.1485 +-934092157,-7843850349.57130038 +294598722,-3542.6 +284554389,5.727146 +90660785,12590.288613 +-99948814,-38076694.3981 +466567142,-9763217822.129028 +1909136587,-8610.078036935181 +1242586043,-4 +\N,1.089120893565337 +1039864870,987601.57 +-466171792,0 +-1681455031,-6.4543 +1755897735,-39.965207 +1585021913,745222.66808954 +448130683,-4302.485366846491 +193709887,0.8 +-424713789,0.48 +1585021913,607.22747 +-1250662632,5454127198.951479 +294598722,-9377326244.444 +193709887,-19889.83 +1039864870,0.7 +1242586043,-749975924224.63 +-1250662632,-544.554649 +-1740848088,-9.157 +-369457052,7.7 +-369457052,560.11907883090455 +90660785,-4564.517185 +466567142,-58810.60586 +466567142,196.5785295398584 +1738753776,1525.280459649262 +1816559437,-1035.7009 +-1490239076,92253.232096 +1039864870,94.04 +560745412,678.25 +-466171792,4227.5344 +1561921421,53050.55 +-99948814,-96386.438 +1519948464,152 +1719049112,-7888197 +-793950320,-16 +-466171792,69.9 +1738753776,-99817635066320.2416 +1091836730,0.02 +891262439,-0.04 +452719211,3020.2938930744636 +-2048404259,3939387044.1 +698032489,-330457.4292625839 +-1197550983,-0.5588796922 +-2123273881,-55.89198 +-2048404259,-0.3222960446251 +1585021913,-5762331.06697112 +1785750809,47443.115 +1909136587,181.07681535944 +1801735854,-1760956929364.267 +\N,4.26165227 +1801735854,-438541294.7 +150678276,-8278 +1479580778,92077343080.7 +1091836730,-5017.14 +193709887,-0.5663 +-1681455031,-11105.372477 +-1250662632,93104 +-1197550983,0.1 +\N,682070836.2649603 +-1197550983,71852.8338674412613 +1561921421,-5.405 +-1740848088,0.506394259 +150678276,15989394.8436 +-793950320,-0.1 +-1740848088,901.441 +-477147437,6 +-1264372462,0.883 +-2123273881,3.959 +-1264372462,-6993985240226 +-1264372462,-899 +-243940373,-97176129669.654953 +-243940373,-583.258 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_long_small_1d.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_long_small_1d.txt b/data/files/fullouter_long_small_1d.txt new file mode 100644 index 0000000..9778d3f --- /dev/null +++ b/data/files/fullouter_long_small_1d.txt @@ -0,0 +1,39 @@ +533298451 +1164387380 +1614287784 +1635405412 +-1912571616 +-894799664 +-1210744742 +-1014271154 +-747044796 +-1003639073 +436878811 +-1323620496 +-1379355738 +-1712018127 +246169862 +1431997749 +670834064 +1780951928 +-707688773 +1997943409 +1372592319 +-932176731 +162858059 +-683339273 +-497171161 +699863556 +1685473722 +41376947 +-1036083124 +1825107160 +-2038654700 +2119085509 +260588085 +-1792852276 +1831520491 +103640700 +\N +699007128 +1840266070 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_multikey_big_1a.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_multikey_big_1a.txt b/data/files/fullouter_multikey_big_1a.txt new file mode 100644 index 0000000..fe38c7b --- /dev/null +++ b/data/files/fullouter_multikey_big_1a.txt @@ -0,0 +1,13 @@ +22767,-1969080993 +-17582,-1730236061 +3556,\N +-17582,1082230084 +-17582,827141667 +1499,371855128 +-17582,9637312 +\N,1082230084 +-6131,-1969080993 +3556,-1969080993 +\N,\N +-18222,-1969080993 +-17582,267529350 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_multikey_big_1a_nonull.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_multikey_big_1a_nonull.txt b/data/files/fullouter_multikey_big_1a_nonull.txt new file mode 100644 index 0000000..40e84b0 --- /dev/null +++ b/data/files/fullouter_multikey_big_1a_nonull.txt @@ -0,0 +1,10 @@ +-17582,1082230084 +22767,-1969080993 +-17582,827141667 +-17582,-1730236061 +3556,-1969080993 +-6131,-1969080993 +-18222,-1969080993 +1499,371855128 +-17582,267529350 +-17582,9637312 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_multikey_big_1b.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_multikey_big_1b.txt b/data/files/fullouter_multikey_big_1b.txt new file mode 100644 index 0000000..40cfb9a --- /dev/null +++ b/data/files/fullouter_multikey_big_1b.txt @@ -0,0 +1,17 @@ +2061-12-19 22:10:32.000628309,21635,ANCO +\N,21635,ANCO +2686-05-23 07:46:46.565832918,13212,NCYBDW +2082-07-14 04:00:40.695380469,12556,NCYBDW +2188-06-04 15:03:14.963259704,9468,AAA +2608-02-23 23:44:02.546440891,26184,NCYBDW +2093-04-10 23:36:54.846,\N,\N +2898-10-01 22:27:02.000871113,10361,NCYBDW +2306-06-21 11:02:00.143124239,1446,\N +\N,-6909,\N +\N,\N,\N +2306-06-21 11:02:00.143124239,-6909,NCYBDW +2093-04-10 23:36:54.846,1446,GHZVPWFO +\N,\N,CCWYD +2686-05-23 07:46:46.565832918,\N,GHZVPWFO +2093-04-10 23:36:54.846,28996,Q +2299-11-15 16:41:30.401,-31077,NCYBDW http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_multikey_small_1a.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_multikey_small_1a.txt b/data/files/fullouter_multikey_small_1a.txt new file mode 100644 index 0000000..4e0742c --- /dev/null +++ b/data/files/fullouter_multikey_small_1a.txt @@ -0,0 +1,92 @@ +23015,258882280 +23015,-276888585 +21186,-586336015 +-22311,-2055239583 +3412,-1249487623 +\N,1082230084 +20156,-1618478138 +-17788,-738743861 +-24206,-1456409156 +30353,2044473567 +20969,-1995259010 +-23457,-63842445 +3412,-2081156563 +-6131,-1969080993 +23015,-252525791 +30353,1364268303 +23015,564751472 +15404,1078466156 +4586,-586336015 +-4117,-1386947816 +-26894,-63842445 +-17788,-1361776766 +-7386,-2112062470 +23015,-1893013623 +30353,1241923267 +-24206,641361618 +-28129,-2055239583 +-20125,-1995259010 +16166,931172175 +31443,-1968665833 +-28313,837320573 +11460,1078466156 +15061,-63842445 +13672,-63842445 +14400,-825652334 +-7386,100736776 +26944,-1995259010 +-11868,97203778 +12089,-63842445 +-28137,-63842445 +3412,1253976194 +-980,2009785365 +16696,-63842445 +-11868,930596435 +4902,1078466156 +-17582,267529350 +-12252,964377504 +20156,963883665 +-11868,1658440922 +4779,-1995259010 +-7386,-1635102480 +-28313,51228026 +-11868,1052120431 +-980,-270600267 +-20900,1078466156 +\N,\N +20156,1165375499 +30353,-1507157031 +3412,-1196037018 +22934,-1695419330 +30353,105613996 +-17788,-872691214 +-980,-333603940 +30353,-1011627089 +-11868,-3536499 +-2407,1078466156 +23015,-217613200 +-28313,-706104224 +-980,712692345 +-11868,1456809245 +-17788,528419995 +-11868,-915441041 +-980,628784462 +30353,-1007182618 +23015,-696928205 +-980,356970043 +23015,-893234501 +-980,-465544127 +-5734,1078466156 +-980,-801821285 +26738,-2055239583 +8177,-1995259010 +-11868,1318114822 +3890,1411429004 +-6061,-586336015 +3412,-2132472060 +-15212,-2055239583 +-12252,1956403781 +5957,-1995259010 +-1787,-63842445 +20156,1855042153 +-980,1310479628 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_multikey_small_1a_nonull.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_multikey_small_1a_nonull.txt b/data/files/fullouter_multikey_small_1a_nonull.txt new file mode 100644 index 0000000..2a8b9a1 --- /dev/null +++ b/data/files/fullouter_multikey_small_1a_nonull.txt @@ -0,0 +1,90 @@ +16696,-63842445 +4586,-586336015 +26738,-2055239583 +-17788,-738743861 +-28313,-706104224 +-23457,-63842445 +-20900,1078466156 +-12252,964377504 +-28313,51228026 +-11868,-3536499 +11460,1078466156 +26944,-1995259010 +20156,1855042153 +-11868,97203778 +15061,-63842445 +-17788,528419995 +-26894,-63842445 +-28313,837320573 +20156,963883665 +-15212,-2055239583 +5957,-1995259010 +30353,-1011627089 +3890,1411429004 +-980,-333603940 +13672,-63842445 +-980,628784462 +23015,-252525791 +-11868,1052120431 +-980,356970043 +23015,-217613200 +-6061,-586336015 +-5734,1078466156 +-11868,1318114822 +23015,258882280 +-2407,1078466156 +12089,-63842445 +3412,-2132472060 +-28129,-2055239583 +-980,-270600267 +16166,931172175 +-7386,100736776 +4902,1078466156 +20969,-1995259010 +22934,-1695419330 +3412,-1249487623 +3412,1253976194 +21186,-586336015 +8177,-1995259010 +-7386,-1635102480 +-11868,1456809245 +-20125,-1995259010 +-980,-801821285 +-980,1310479628 +23015,564751472 +23015,-893234501 +4779,-1995259010 +-980,2009785365 +-24206,641361618 +30353,-1507157031 +14400,-825652334 +3412,-2081156563 +20156,-1618478138 +31443,-1968665833 +-22311,-2055239583 +30353,1241923267 +-11868,930596435 +-17788,-1361776766 +-24206,-1456409156 +-7386,-2112062470 +30353,1364268303 +23015,-1893013623 +-17788,-872691214 +30353,2044473567 +-28137,-63842445 +30353,105613996 +-6131,-1969080993 +-17582,267529350 +23015,-276888585 +-12252,1956403781 +23015,-696928205 +-11868,1658440922 +-1787,-63842445 +-11868,-915441041 +-980,-465544127 +30353,-1007182618 +-980,712692345 +20156,1165375499 +3412,-1196037018 +15404,1078466156 +-4117,-1386947816 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_multikey_small_1b.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_multikey_small_1b.txt b/data/files/fullouter_multikey_small_1b.txt new file mode 100644 index 0000000..b56a3f7 --- /dev/null +++ b/data/files/fullouter_multikey_small_1b.txt @@ -0,0 +1,118 @@ +2304-12-15 15:31:16,11101,YJCKKCR,-0.2 +2018-11-25 22:27:55.84,-12202,VBDBM,7506645.9537 +1957-03-06 09:57:31,-26373,NXLNNSO,2 +2332-06-14 07:02:42.32,-26373,XFFFDTQ,56845106806308.9 +2535-03-01 05:04:49.000525883,23663,ALIQKNXHE,-0.1665691 +2629-04-07 01:54:11,-6776,WGGFVFTW,6.8012851708 +2266-09-26 06:27:29.000284762,20223,EDYJJN,14 +2969-01-23 14:08:04.000667259,-18138,VDPN,8924831210.42768019 +2861-05-27 07:13:01.000848622,-19598,WKPXNLXS,29399 +2301-06-03 17:16:19,15332,ZVEUKC,0.5 +1980-09-13 19:57:15,\N,M,57650.7723 +2304-12-15 15:31:16,1301,T,-0.8 +2461-03-09 09:54:45.000982385,-16454,ZSMB,-991.43605 +2044-05-02 07:00:03.35,-8751,ZSMB,-453797242.029791752 +2409-09-23 10:33:27,2638,XSXR,-9926693851 +1941-10-16 02:19:36.000423663,-24459,AO,-821445414.4579712 +2512-10-06 03:03:03,-3465,VZQ,-49.51219 +2971-02-14 09:13:19,-16605,BVACIRP,-5.751278023 +2075-10-25 20:32:40.000792874,\N,\N,226612651968.36076 +2073-03-21 15:32:57.617920888,26425,MPRACIRYW,5 +2969-01-23 14:08:04.000667259,14500,WXLTRFQP,-23.8198 +2898-12-18 03:37:17,-24459,MHNBXPBM,14.23669356238481 +\N,\N,\N,-2207.3 +2391-01-17 15:28:37.00045143,16160,ZVEUKC,771355639420297.133 +2309-01-15 12:43:49,22821,ZMY,40.9 +2340-12-15 05:15:17.133588982,23663,HHTP,33383.8 +2969-01-23 14:08:04.000667259,-8913,UIMQ,9.178 +2145-10-15 06:58:42.831,2638,\N,-9784.82 +2888-05-08 08:36:55.182302102,5786,ZVEUKC,-56082455.033918 +2467-05-11 06:04:13.426693647,23196,EIBSDASR,-8.5548883801 +2829-06-04 08:01:47.836,22771,ZVEUKC,94317.75318 +2938-12-21 23:35:59.498,29362,ZMY,0.88 +2304-12-15 15:31:16,-13125,JFYW,6.086657 +2808-07-09 02:10:11.928498854,-19598,FHFX,0.3 +2083-06-07 09:35:19.383,-26373,MR,-394.0867 +2686-05-23 07:46:46.565832918,13212,NCYBDW,-917116793.4 +2969-01-23 14:08:04.000667259,-8913,UIMQ,-375994644577.315257 +2338-02-12 09:30:07,20223,CTH,-6154.763054 +2629-04-07 01:54:11,-6776,WGGFVFTW,41.77451507786646 +2242-08-04 07:51:46.905,20223,UCYXACQ,37.7288 +2637-03-12 22:25:46.385,-12923,PPTJPFR,5.4 +2304-12-15 15:31:16,8650,RLNO,0.71351747335 +2688-02-06 20:58:42.000947837,20223,PAIY,67661.735 +\N,\N,\N,-2.4 +2512-10-06 03:03:03,-3465,VZQ,0.4458 +2960-04-12 07:03:42.000366651,20340,CYZYUNSF,-96.3 +2461-03-09 09:54:45.000982385,-16454,ZSMB,-9575827.55396 +2512-10-06 03:03:03,1560,X,-922.6951584107 +2396-04-06 15:39:02.404013577,29661,ZSMB,0.76718326 +2409-09-23 10:33:27,2638,XSXR,0.4 +2969-01-23 14:08:04.000667259,6689,TFGVOGPJF,-0.01 +2333-07-28 09:59:26,23196,RKSK,37872288434740893.5 +2409-09-23 10:33:27,2638,XSXR,-162.95 +2357-05-08 07:09:09.000482799,6226,ZSMB,-472 +2304-12-15 15:31:16,15090,G,-4319470286240016.3 +2304-12-15 15:31:16,1301,T,61.302 +2105-01-04 16:27:45,23100,ZSMB,-83.2328 +2242-08-04 07:51:46.905,20223,UCYXACQ,-0.26149 +2637-03-12 22:25:46.385,-17786,HYEGQ,-84.169614329419 +1931-12-04 11:13:47.269597392,23196,HVJCQMTQL,-9697532.8994 +2897-08-10 15:21:47.09,23663,XYUVBED,6370 +2888-05-08 08:36:55.182302102,5786,ZVEUKC,57.62175257788037 +2145-10-15 06:58:42.831,2638,UANGISEXR,-5996.306 +2462-12-16 23:11:32.633305644,-26373,CB,67.41799 +2396-04-06 15:39:02.404013577,29661,ZSMB,-5151598.347 +2304-12-15 15:31:16,15090,G,975 +2512-10-06 03:03:03,32099,ARNZ,-0.41 +2188-06-04 15:03:14.963259704,9468,AAA,2.75496352 +2512-10-06 03:03:03,1560,X,761196.522 +2304-12-15 15:31:16,1301,T,2720.8 +1919-06-20 00:16:50.611028595,20223,ZKBC,-23 +2897-08-10 15:21:47.09,23663,XYUVBED,51.7323303273 +2086-04-09 00:03:10,20223,THXNJGFFV,-85184687349898.892 +2238-05-17 19:27:25.519,20223,KQCM,-0.01095 +2086-04-09 00:03:10,20223,THXNJGFFV,482.5383411359219 +2480-10-02 09:31:37.000770961,-26373,NBN,-5875.5197252 +2086-04-09 00:03:10,20223,THXNJGFFV,0.4396861 +2759-11-26 22:19:55.410967136,-27454,ZMY,60.6025797 +2083-06-07 09:35:19.383,-26373,MR,67892053.02376094 +2882-05-20 07:21:25.221299462,23196,U,-9951044 +2971-02-14 09:13:19,-16605,BVACIRP,-27394351.3 +2512-10-06 03:03:03,24313,QBHUG,-8423.151573236 +2882-05-20 07:21:25.221299462,23196,U,-4244.926206619 +1905-04-20 13:42:25.000469776,2638,KAUUFF,7 +2410-05-03 13:44:56,2638,PHOR,-769088.176482 +2668-06-25 07:12:37.000970744,2638,TJE,-2.7796827 +2969-01-23 14:08:04.000667259,-32485,AGEPWWLJF,-48431309405.652522 +2410-05-03 13:44:56,2638,PHOR,93262.914526611 +2512-10-06 03:03:03,13195,CRJ,14 +2018-11-25 22:27:55.84,-12202,VBDBM,98790.713907420831 +2304-12-15 15:31:16,8650,RLNO,-0.4355 +2071-07-21 20:02:32.000250697,2638,NRUV,-66198.351092 +2525-05-12 15:59:35,-24459,SAVRGA,53106747151.8633 +2637-03-12 22:25:46.385,21841,CXTI,749563668434009.65 +2018-11-25 22:27:55.84,-22419,LOTLS,342.3726040228584 +2637-03-12 22:25:46.385,21841,CXTI,7362887891522.3782 +2038-10-12 09:15:33.000539653,-19598,YKNIAJW,-642807895924.66 +2957-05-07 10:41:46,20223,OWQT,-586953.153681 +2304-12-15 15:31:16,11101,YJCKKCR,1279917802.42 +2355-09-23 19:52:34.638084141,-19598,H,92.15 +2960-04-12 07:03:42.000366651,20340,CYZYUNSF,2.1577659 +2355-09-23 19:52:34.638084141,-19598,H,74179461.880493 +2969-01-23 14:08:04.000667259,-8913,UIMQ,-81 +\N,-12914,ZVEUKC,221 +2743-12-27 05:16:19.000573579,-12914,ZVEUKC,-811984611.5178497 +1957-02-01 14:00:29.000548421,-16085,ZVEUKC,-2312.8149 +2201-07-05 17:22:06.084206844,-24459,UBGT,1.5069483282 +2461-03-09 09:54:45.000982385,-16454,ZSMB,8694.89 +2169-04-02 06:30:32,23855,PDVQATOS,-1515597428 +2304-12-15 15:31:16,30285,GSJPSIYOU,0.2 +2913-07-17 15:06:58.041,-10206,\N,-0.2 +2169-04-02 06:30:32,23855,PDVQATOS,-4016.9608 +2759-11-26 22:19:55.410967136,-27454,ZMY,368 +2073-03-21 15:32:57.617920888,26425,MPRACIRYW,726945733.4193 +2304-12-15 15:31:16,11101,YJCKKCR,-0.5 +2462-12-16 23:11:32.633305644,-26373,CB,-582687 +2357-05-08 07:09:09.000482799,6226,ZSMB,-32.46 +2304-12-15 15:31:16,12587,OPW,-4.59489504 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_string_big_1a.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_string_big_1a.txt b/data/files/fullouter_string_big_1a.txt new file mode 100644 index 0000000..1cbcd05 --- /dev/null +++ b/data/files/fullouter_string_big_1a.txt @@ -0,0 +1,13 @@ +FTWURVH +QNCYBDW +UA +WXHJ +\N +WXHJ +PXLD +WXHJ +PXLD +WXHJ +WXHJ +MXGDMBD +PXLD http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_string_big_1a_nonull.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_string_big_1a_nonull.txt b/data/files/fullouter_string_big_1a_nonull.txt new file mode 100644 index 0000000..a6566f2 --- /dev/null +++ b/data/files/fullouter_string_big_1a_nonull.txt @@ -0,0 +1,12 @@ +WXHJ +WXHJ +FTWURVH +MXGDMBD +UA +WXHJ +QNCYBDW +PXLD +PXLD +WXHJ +PXLD +WXHJ http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_string_big_1a_old.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_string_big_1a_old.txt b/data/files/fullouter_string_big_1a_old.txt new file mode 100644 index 0000000..1fa51ad --- /dev/null +++ b/data/files/fullouter_string_big_1a_old.txt @@ -0,0 +1,13 @@ +WXHJ +WXHJ +WXHJ +WXHJ +WXHJ +QNCYBDW +PXLD +PXLD +PXLD +UA +\N +FTWURVH +MXGDMBD http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_string_small_1a.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_string_small_1a.txt b/data/files/fullouter_string_small_1a.txt new file mode 100644 index 0000000..f223da0 --- /dev/null +++ b/data/files/fullouter_string_small_1a.txt @@ -0,0 +1,38 @@ +BDBMW,2278-04-27,2101-02-21 08:53:34.692 +FROPIK,2023-02-28,2467-05-11 06:04:13.426693647 +GOYJHW,1976-03-06,2805-07-10 10:51:57.00083302 +MXGDMBD,1880-11-01,2765-10-06 13:28:17.000688592 +CQMTQLI,2031-09-13,1927-02-13 08:39:25.000919094 +,1985-01-22,2111-01-10 15:44:28 +IOQIDQBHU,2198-02-08,2073-03-21 15:32:57.617920888 +GSJPSIYOU,1948-07-17,2006-09-24 16:01:24.000239251 +\N,1865-11-08,2893-04-07 07:36:12 +BEP,2206-08-10,2331-10-09 10:59:51 +NADANUQMW,2037-10-19,2320-04-26 18:50:25.000426922 +\N,2250-04-22,2548-03-21 08:23:13.133573801 +ATZJTPECF,1829-10-16,2357-05-08 07:09:09.000482799 +IWEZJHKE,\N,\N +AARNZRVZQ,2002-10-23,2525-05-12 15:59:35 +BEP,2141-02-19,2521-06-09 01:20:07.121 +AARNZRVZQ,2000-11-13,2309-06-05 19:54:13 +LOTLS,1957-11-09,2092-06-07 06:42:30.000538454 +FROPIK,2124-10-01,2974-07-06 12:05:08.000146048 +KL,1980-09-22,2073-08-25 11:51:10.318 +\N,1915-02-22,2554-10-27 09:34:30 +WNGFTTY,1843-06-10,2411-01-28 20:03:59 +VNRXWQ,1883-02-06,2287-07-17 16:46:58.287 +QTSRKSKB,2144-01-13,2627-12-20 03:38:53.000389266 +GOYJHW,1959-04-27,\N +LOTLS,2099-08-04,2181-01-25 01:04:25.000030055 +CQMTQLI,2090-11-13,2693-03-17 16:19:55.82 +VNRXWQ,2276-11-16,2072-08-16 17:45:47.48349887 +LOTLS,2126-09-16,1977-12-15 15:28:56 +FTWURVH,1976-03-10,2683-11-22 13:07:04.66673556 +,2021-02-21,2802-04-21 18:48:18.5933838 +ZNOUDCR,\N,1988-04-23 08:40:21 +FROPIK,2214-02-09,1949-08-18 17:14:38.000703738 +SDA,2196-04-12,2462-10-26 19:28:12.733 +WNGFTTY,2251-08-16,2649-12-21 18:30:42.498 +GOYJHW,1993-04-07,1950-05-04 09:28:22.000114784 +FYW,1807-03-20,2305-08-17 01:32:44 +ATZJTPECF,2217-10-22,2808-10-20 16:01:24.558 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_string_small_1a_nonull.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_string_small_1a_nonull.txt b/data/files/fullouter_string_small_1a_nonull.txt new file mode 100644 index 0000000..6b97ef4 --- /dev/null +++ b/data/files/fullouter_string_small_1a_nonull.txt @@ -0,0 +1,35 @@ +LOTLS,2126-09-16,1977-12-15 15:28:56 +MXGDMBD,1880-11-01,2765-10-06 13:28:17.000688592 +WNGFTTY,2251-08-16,2649-12-21 18:30:42.498 +QTSRKSKB,2144-01-13,2627-12-20 03:38:53.000389266 +AARNZRVZQ,2002-10-23,2525-05-12 15:59:35 +BEP,2141-02-19,2521-06-09 01:20:07.121 +ZNOUDCR,\N,1988-04-23 08:40:21 +FROPIK,2023-02-28,2467-05-11 06:04:13.426693647 +GOYJHW,1993-04-07,1950-05-04 09:28:22.000114784 +CQMTQLI,2090-11-13,2693-03-17 16:19:55.82 +BDBMW,2278-04-27,2101-02-21 08:53:34.692 +AARNZRVZQ,2000-11-13,2309-06-05 19:54:13 +FYW,1807-03-20,2305-08-17 01:32:44 +,2021-02-21,2802-04-21 18:48:18.5933838 +VNRXWQ,1883-02-06,2287-07-17 16:46:58.287 +FROPIK,2124-10-01,2974-07-06 12:05:08.000146048 +LOTLS,2099-08-04,2181-01-25 01:04:25.000030055 +BEP,2206-08-10,2331-10-09 10:59:51 +WNGFTTY,1843-06-10,2411-01-28 20:03:59 +LOTLS,1957-11-09,2092-06-07 06:42:30.000538454 +CQMTQLI,2031-09-13,1927-02-13 08:39:25.000919094 +GOYJHW,1976-03-06,2805-07-10 10:51:57.00083302 +,1985-01-22,2111-01-10 15:44:28 +SDA,2196-04-12,2462-10-26 19:28:12.733 +ATZJTPECF,1829-10-16,2357-05-08 07:09:09.000482799 +GOYJHW,1959-04-27,\N +FTWURVH,1976-03-10,2683-11-22 13:07:04.66673556 +KL,1980-09-22,2073-08-25 11:51:10.318 +ATZJTPECF,2217-10-22,2808-10-20 16:01:24.558 +NADANUQMW,2037-10-19,2320-04-26 18:50:25.000426922 +FROPIK,2214-02-09,1949-08-18 17:14:38.000703738 +IWEZJHKE,\N,\N +GSJPSIYOU,1948-07-17,2006-09-24 16:01:24.000239251 +IOQIDQBHU,2198-02-08,2073-03-21 15:32:57.617920888 +VNRXWQ,2276-11-16,2072-08-16 17:45:47.48349887 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/data/files/fullouter_string_small_1a_old.txt ---------------------------------------------------------------------- diff --git a/data/files/fullouter_string_small_1a_old.txt b/data/files/fullouter_string_small_1a_old.txt new file mode 100644 index 0000000..505c403 --- /dev/null +++ b/data/files/fullouter_string_small_1a_old.txt @@ -0,0 +1,38 @@ +,2021-02-21,2802-04-21 18:48:18.5933838 +,1985-01-22,2111-01-10 15:44:28 +VNRXWQ,1883-02-06,2287-07-17 16:46:58.287 +VNRXWQ,2276-11-16,2072-08-16 17:45:47.48349887 +KL,1980-09-22,2073-08-25 11:51:10.318 +FYW,1807-03-20,2305-08-17 01:32:44 +WNGFTTY,2251-08-16,2649-12-21 18:30:42.498 +WNGFTTY,1843-06-10,2411-01-28 20:03:59 +FTWURVH,1976-03-10,2683-11-22 13:07:04.66673556 +CQMTQLI,2031-09-13,1927-02-13 08:39:25.000919094 +CQMTQLI,2090-11-13,2693-03-17 16:19:55.82 +BEP,2141-02-19,2521-06-09 01:20:07.121 +BEP,2206-08-10,2331-10-09 10:59:51 +FROPIK,2023-02-28,2467-05-11 06:04:13.426693647 +FROPIK,2214-02-09,1949-08-18 17:14:38.000703738 +FROPIK,2124-10-01,2974-07-06 12:05:08.000146048 +SDA,2196-04-12,2462-10-26 19:28:12.733 +ATZJTPECF,2217-10-22,2808-10-20 16:01:24.558 +ATZJTPECF,1829-10-16,2357-05-08 07:09:09.000482799 +MXGDMBD,1880-11-01,2765-10-06 13:28:17.000688592 +IWEZJHKE,\N,\N +NADANUQMW,2037-10-19,2320-04-26 18:50:25.000426922 +GOYJHW,1993-04-07,1950-05-04 09:28:22.000114784 +GOYJHW,1976-03-06,2805-07-10 10:51:57.00083302 +GOYJHW,1959-04-27,\N +QTSRKSKB,2144-01-13,2627-12-20 03:38:53.000389266 +IOQIDQBHU,2198-02-08,2073-03-21 15:32:57.617920888 +ZNOUDCR,\N,1988-04-23 08:40:21 +AARNZRVZQ,2000-11-13,2309-06-05 19:54:13 +AARNZRVZQ,2002-10-23,2525-05-12 15:59:35 +BDBMW,2278-04-27,2101-02-21 08:53:34.692 +\N,1865-11-08,2893-04-07 07:36:12 +\N,2250-04-22,2548-03-21 08:23:13.133573801 +\N,1915-02-22,2554-10-27 09:34:30 +LOTLS,1957-11-09,2092-06-07 06:42:30.000538454 +LOTLS,2126-09-16,1977-12-15 15:28:56 +LOTLS,2099-08-04,2181-01-25 01:04:25.000030055 +GSJPSIYOU,1948-07-17,2006-09-24 16:01:24.000239251 http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java ---------------------------------------------------------------------- diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java index af446db..48f85f4 100644 --- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java +++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/AbstractMapJoin.java @@ -13,21 +13,25 @@ */ package org.apache.hive.benchmark.vectorization.mapjoin; +import java.util.ArrayList; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.tez.ObjectCache; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer; import org.apache.hadoop.hive.ql.exec.util.collectoroperator.CountCollectorTestOperator; import org.apache.hadoop.hive.ql.exec.util.collectoroperator.CountVectorCollectorTestOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomBatchSource; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig.MapJoinTestImplementation; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestData; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription; +import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig.CreateMapJoinResult; +import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription.MapJoinPlanVariation; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription.SmallTableGenerationParameters; -import org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerateUtil; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.VectorMapJoinVariation; @@ -74,30 +78,31 @@ public abstract class AbstractMapJoin { } protected void setupMapJoin(HiveConf hiveConf, long seed, int rowCount, - VectorMapJoinVariation vectorMapJoinVariation, MapJoinTestImplementation mapJoinImplementation, - String[] bigTableColumnNames, TypeInfo[] bigTableTypeInfos, int[] bigTableKeyColumnNums, - String[] smallTableValueColumnNames, TypeInfo[] smallTableValueTypeInfos, - int[] bigTableRetainColumnNums, - int[] smallTableRetainKeyColumnNums, int[] smallTableRetainValueColumnNums, - SmallTableGenerationParameters smallTableGenerationParameters) throws Exception { + VectorMapJoinVariation vectorMapJoinVariation, MapJoinTestImplementation mapJoinImplementation, + String[] bigTableColumnNames, TypeInfo[] bigTableTypeInfos, + int[] bigTableKeyColumnNums, + String[] smallTableValueColumnNames, TypeInfo[] smallTableValueTypeInfos, + int[] bigTableRetainColumnNums, + int[] smallTableRetainKeyColumnNums, int[] smallTableRetainValueColumnNums, + SmallTableGenerationParameters smallTableGenerationParameters) throws Exception { this.vectorMapJoinVariation = vectorMapJoinVariation; this.mapJoinImplementation = mapJoinImplementation; testDesc = new MapJoinTestDescription( hiveConf, vectorMapJoinVariation, - bigTableColumnNames, bigTableTypeInfos, + bigTableTypeInfos, bigTableKeyColumnNums, - smallTableValueColumnNames, smallTableValueTypeInfos, - bigTableRetainColumnNums, - smallTableRetainKeyColumnNums, smallTableRetainValueColumnNums, - smallTableGenerationParameters); + smallTableValueTypeInfos, + smallTableRetainKeyColumnNums, + smallTableGenerationParameters, + MapJoinPlanVariation.DYNAMIC_PARTITION_HASH_JOIN); // Prepare data. Good for ANY implementation variation. - testData = new MapJoinTestData(rowCount, testDesc, seed, seed * 10); + testData = new MapJoinTestData(rowCount, testDesc, seed); ObjectRegistryImpl objectRegistry = new ObjectRegistryImpl(); ObjectCache.setupObjectRegistry(objectRegistry); - + operator = setupBenchmarkImplementation( mapJoinImplementation, testDesc, testData); @@ -108,15 +113,21 @@ public abstract class AbstractMapJoin { */ if (!isVectorOutput) { - bigTableRows = VectorBatchGenerateUtil.generateRowObjectArray( - testDesc.bigTableKeyTypeInfos, testData.getBigTableBatchStream(), - testData.getBigTableBatch(), testDesc.outputObjectInspectors); + bigTableRows = testData.getBigTableBatchSource().getRandomRows(); } else { - bigTableBatches = VectorBatchGenerateUtil.generateBatchArray( - testData.getBigTableBatchStream(), testData.getBigTableBatch()); - + ArrayList<VectorizedRowBatch> bigTableBatchList = new ArrayList<VectorizedRowBatch>(); + VectorRandomBatchSource batchSource = testData.getBigTableBatchSource(); + batchSource.resetBatchIteration(); + while (true) { + VectorizedRowBatch batch = testData.createBigTableBatch(testDesc); + if (!batchSource.fillNextBatch(batch)) { + break; + } + bigTableBatchList.add(batch); + } + bigTableBatches = bigTableBatchList.toArray(new VectorizedRowBatch[0]); } } @@ -131,7 +142,6 @@ public abstract class AbstractMapJoin { MapJoinTestData testData) throws Exception { - // UNDONE: Parameterize for implementation variation? MapJoinDesc mapJoinDesc = MapJoinTestConfig.createMapJoinDesc(testDesc); final boolean isVectorOutput = isVectorOutput(mapJoinImplementation); @@ -141,9 +151,19 @@ public abstract class AbstractMapJoin { (!isVectorOutput ? new CountCollectorTestOperator() : new CountVectorCollectorTestOperator()); - MapJoinOperator operator = + CreateMapJoinResult createMapJoinResult = MapJoinTestConfig.createMapJoinImplementation( - mapJoinImplementation, testDesc, testCollectorOperator, testData, mapJoinDesc); + mapJoinImplementation, testDesc, testData, mapJoinDesc, + /* shareMapJoinTableContainer */ null); + MapJoinOperator operator = createMapJoinResult.mapJoinOperator; + MapJoinTableContainer mapJoinTableContainer = createMapJoinResult.mapJoinTableContainer; + + // Invoke initializeOp methods. + operator.initialize(testDesc.hiveConf, testDesc.inputObjectInspectors); + + // Fixup the mapJoinTables. + operator.setTestMapJoinTableContainer(1, mapJoinTableContainer, null); + return operator; } http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java ---------------------------------------------------------------------- diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java index c9da92a..aa88297 100644 --- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java +++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBenchBase.java @@ -59,7 +59,8 @@ public abstract class MapJoinMultiKeyBenchBase extends AbstractMapJoin { setupMapJoin(hiveConf, seed, rowCount, vectorMapJoinVariation, mapJoinImplementation, - bigTableColumnNames, bigTableTypeInfos, bigTableKeyColumnNums, + bigTableColumnNames, bigTableTypeInfos, + bigTableKeyColumnNums, smallTableValueColumnNames, smallTableValueTypeInfos, bigTableRetainColumnNums, smallTableRetainKeyColumnNums, smallTableRetainValueColumnNums, http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java ---------------------------------------------------------------------- diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java index a6b4719..60b2890 100644 --- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java +++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneLongKeyBenchBase.java @@ -57,7 +57,8 @@ public abstract class MapJoinOneLongKeyBenchBase extends AbstractMapJoin { setupMapJoin(hiveConf, seed, rowCount, vectorMapJoinVariation, mapJoinImplementation, - bigTableColumnNames, bigTableTypeInfos, bigTableKeyColumnNums, + bigTableColumnNames, bigTableTypeInfos, + bigTableKeyColumnNums, smallTableValueColumnNames, smallTableValueTypeInfos, bigTableRetainColumnNums, smallTableRetainKeyColumnNums, smallTableRetainValueColumnNums, http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java ---------------------------------------------------------------------- diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java index 1b31038..937ede1 100644 --- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java +++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinOneStringKeyBenchBase.java @@ -57,7 +57,8 @@ public abstract class MapJoinOneStringKeyBenchBase extends AbstractMapJoin { setupMapJoin(hiveConf, seed, rowCount, vectorMapJoinVariation, mapJoinImplementation, - bigTableColumnNames, bigTableTypeInfos, bigTableKeyColumnNums, + bigTableColumnNames, bigTableTypeInfos, + bigTableKeyColumnNums, smallTableValueColumnNames, smallTableValueTypeInfos, bigTableRetainColumnNums, smallTableRetainKeyColumnNums, smallTableRetainValueColumnNums, http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 0e071fb..64e8183 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -512,6 +512,7 @@ minillaplocal.query.files=\ explainanalyze_2.q,\ explainuser_1.q,\ explainuser_4.q,\ + fullouter_mapjoin_1_optimized.q,\ groupby2.q,\ groupby_groupingset_bug.q,\ hybridgrace_hashjoin_1.q,\ @@ -760,6 +761,10 @@ minillaplocal.query.files=\ vector_create_struct_table.q,\ vector_decimal_2.q,\ vector_decimal_udf.q,\ + vector_full_outer_join.q,\ + vector_fullouter_mapjoin_1_fast.q,\ + vector_fullouter_mapjoin_1_optimized.q,\ + vector_fullouter_mapjoin_1_optimized_passthru.q,\ vector_groupby_cube1.q,\ vector_groupby_grouping_id1.q,\ vector_groupby_grouping_id2.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java index 02a67cb..3762ee5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java @@ -790,7 +790,16 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends } private boolean hasAnyFiltered(int alias, List<Object> row) { - return row == dummyObj[alias] || hasFilter(alias) && JoinUtil.hasAnyFiltered(getFilterTag(row)); + if (row == dummyObj[alias]) { + return true; + } + if (hasFilter(alias) && row != null) { + ShortWritable shortWritable = (ShortWritable) row.get(row.size() - 1); + if (shortWritable != null) { + return JoinUtil.hasAnyFiltered(shortWritable.get()); + } + } + return false; } protected final boolean hasFilter(int alias) { http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index 65ec4b2..2f20dd4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -732,6 +732,73 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { return outputPlan(work, out, extended, jsonOutput, indent, ""); } + private boolean isInvokeVectorization(Vectorization vectorization) { + + boolean invokeFlag = true; // Assume. + + // The EXPLAIN VECTORIZATION option was specified. + final boolean desireOnly = this.work.isVectorizationOnly(); + final VectorizationDetailLevel desiredVecDetailLevel = + this.work.isVectorizationDetailLevel(); + + switch (vectorization) { + case NON_VECTORIZED: + // Display all non-vectorized leaf objects unless ONLY. + if (desireOnly) { + invokeFlag = false; + } + break; + case SUMMARY: + case OPERATOR: + case EXPRESSION: + case DETAIL: + if (vectorization.rank < desiredVecDetailLevel.rank) { + // This detail not desired. + invokeFlag = false; + } + break; + case SUMMARY_PATH: + case OPERATOR_PATH: + if (desireOnly) { + if (vectorization.rank < desiredVecDetailLevel.rank) { + // Suppress headers and all objects below. + invokeFlag = false; + } + } + break; + default: + throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization); + } + + return invokeFlag; + } + + private boolean isInvokeNonVectorization(Vectorization vectorization) { + + boolean invokeFlag = true; // Assume. + + // Do not display vectorization objects. + switch (vectorization) { + case SUMMARY: + case OPERATOR: + case EXPRESSION: + case DETAIL: + invokeFlag = false; + break; + case NON_VECTORIZED: + // No action. + break; + case SUMMARY_PATH: + case OPERATOR_PATH: + // Always include headers since they contain non-vectorized objects, too. + break; + default: + throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization); + } + + return invokeFlag; + } + @VisibleForTesting JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception { @@ -756,65 +823,17 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { if (extended) { invokeFlag = Level.EXTENDED.in(xpl_note.explainLevels()); } else { - invokeFlag = Level.DEFAULT.in(xpl_note.explainLevels()); + invokeFlag = + Level.DEFAULT.in(xpl_note.explainLevels()) || + (this.work != null && this.work.isDebug() && Level.DEBUG.in(xpl_note.explainLevels())); } } if (invokeFlag) { Vectorization vectorization = xpl_note.vectorization(); if (this.work != null && this.work.isVectorization()) { - - // The EXPLAIN VECTORIZATION option was specified. - final boolean desireOnly = this.work.isVectorizationOnly(); - final VectorizationDetailLevel desiredVecDetailLevel = - this.work.isVectorizationDetailLevel(); - - switch (vectorization) { - case NON_VECTORIZED: - // Display all non-vectorized leaf objects unless ONLY. - if (desireOnly) { - invokeFlag = false; - } - break; - case SUMMARY: - case OPERATOR: - case EXPRESSION: - case DETAIL: - if (vectorization.rank < desiredVecDetailLevel.rank) { - // This detail not desired. - invokeFlag = false; - } - break; - case SUMMARY_PATH: - case OPERATOR_PATH: - if (desireOnly) { - if (vectorization.rank < desiredVecDetailLevel.rank) { - // Suppress headers and all objects below. - invokeFlag = false; - } - } - break; - default: - throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization); - } + invokeFlag = isInvokeVectorization(vectorization); } else { - // Do not display vectorization objects. - switch (vectorization) { - case SUMMARY: - case OPERATOR: - case EXPRESSION: - case DETAIL: - invokeFlag = false; - break; - case NON_VECTORIZED: - // No action. - break; - case SUMMARY_PATH: - case OPERATOR_PATH: - // Always include headers since they contain non-vectorized objects, too. - break; - default: - throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization); - } + invokeFlag = isInvokeNonVectorization(vectorization); } } if (invokeFlag) { @@ -892,64 +911,18 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { if (extended) { invokeFlag = Level.EXTENDED.in(xpl_note.explainLevels()); } else { - invokeFlag = Level.DEFAULT.in(xpl_note.explainLevels()); + invokeFlag = + Level.DEFAULT.in(xpl_note.explainLevels()) || + (this.work != null && this.work.isDebug() && Level.DEBUG.in(xpl_note.explainLevels())); } } if (invokeFlag) { Vectorization vectorization = xpl_note.vectorization(); - if (this.work != null && this.work.isVectorization()) { - - // The EXPLAIN VECTORIZATION option was specified. - final boolean desireOnly = this.work.isVectorizationOnly(); - final VectorizationDetailLevel desiredVecDetailLevel = - this.work.isVectorizationDetailLevel(); - - switch (vectorization) { - case NON_VECTORIZED: - // Display all non-vectorized leaf objects unless ONLY. - if (desireOnly) { - invokeFlag = false; - } - break; - case SUMMARY: - case OPERATOR: - case EXPRESSION: - case DETAIL: - if (vectorization.rank < desiredVecDetailLevel.rank) { - // This detail not desired. - invokeFlag = false; - } - break; - case SUMMARY_PATH: - case OPERATOR_PATH: - if (desireOnly) { - if (vectorization.rank < desiredVecDetailLevel.rank) { - // Suppress headers and all objects below. - invokeFlag = false; - } - } - break; - default: - throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization); - } - } else { - // Do not display vectorization objects. - switch (vectorization) { - case SUMMARY: - case OPERATOR: - case EXPRESSION: - case DETAIL: - invokeFlag = false; - break; - case NON_VECTORIZED: - // No action. - break; - case SUMMARY_PATH: - case OPERATOR_PATH: - // Always include headers since they contain non-vectorized objects, too. - break; - default: - throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization); + if (invokeFlag) { + if (this.work != null && this.work.isVectorization()) { + invokeFlag = isInvokeVectorization(vectorization); + } else { + invokeFlag = isInvokeNonVectorization(vectorization); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java index a914ce3..1aae142 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java @@ -140,7 +140,7 @@ public class JoinUtil { if (key == (byte) posBigTableAlias) { valueFields.add(null); } else { - valueFields.add(ExprNodeEvaluatorFactory.get(expr, conf)); + valueFields.add(expr == null ? null : ExprNodeEvaluatorFactory.get(expr, conf)); } } outMap[key] = valueFields;
