Author: xuefu
Date: Sat Dec 20 14:55:13 2014
New Revision: 1646998
URL: http://svn.apache.org/r1646998
Log:
HIVE-8640: Support hints of SMBJoin [Spark Branch] (Szehon via Xuefu)
Added:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkJoinHintOptimizer.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out
Modified:
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSortMergeJoinOptimizer.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_10.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_16.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
Modified:
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties
(original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties
Sat Dec 20 14:55:13 2014
@@ -459,14 +459,6 @@ spark.query.files=add_part_multiple.q, \
authorization_admin_almighty1.q, \
auto_join0.q, \
auto_join1.q, \
- auto_join2.q, \
- auto_join3.q, \
- auto_join4.q, \
- auto_join5.q, \
- auto_join6.q, \
- auto_join7.q, \
- auto_join8.q, \
- auto_join9.q, \
auto_join10.q, \
auto_join11.q, \
auto_join12.q, \
@@ -478,6 +470,7 @@ spark.query.files=add_part_multiple.q, \
auto_join18.q, \
auto_join18_multi_distinct.q, \
auto_join19.q, \
+ auto_join2.q, \
auto_join20.q, \
auto_join21.q, \
auto_join22.q, \
@@ -487,9 +480,16 @@ spark.query.files=add_part_multiple.q, \
auto_join27.q, \
auto_join28.q, \
auto_join29.q, \
+ auto_join3.q, \
auto_join30.q, \
auto_join31.q, \
auto_join32.q, \
+ auto_join4.q, \
+ auto_join5.q, \
+ auto_join6.q, \
+ auto_join7.q, \
+ auto_join8.q, \
+ auto_join9.q, \
auto_join_filters.q, \
auto_join_nulls.q, \
auto_join_reordering_values.q, \
@@ -498,6 +498,12 @@ spark.query.files=add_part_multiple.q, \
auto_join_without_localtask.q, \
auto_smb_mapjoin_14.q, \
auto_sortmerge_join_1.q, \
+ auto_sortmerge_join_10.q, \
+ auto_sortmerge_join_12.q, \
+ auto_sortmerge_join_13.q, \
+ auto_sortmerge_join_14.q, \
+ auto_sortmerge_join_15.q, \
+ auto_sortmerge_join_16.q, \
auto_sortmerge_join_2.q, \
auto_sortmerge_join_3.q, \
auto_sortmerge_join_4.q, \
@@ -506,12 +512,6 @@ spark.query.files=add_part_multiple.q, \
auto_sortmerge_join_7.q, \
auto_sortmerge_join_8.q, \
auto_sortmerge_join_9.q, \
- auto_sortmerge_join_10.q, \
- auto_sortmerge_join_12.q, \
- auto_sortmerge_join_13.q, \
- auto_sortmerge_join_14.q, \
- auto_sortmerge_join_15.q, \
- auto_sortmerge_join_16.q, \
avro_compression_enabled_native.q, \
avro_decimal_native.q, \
avro_joins.q, \
@@ -519,7 +519,19 @@ spark.query.files=add_part_multiple.q, \
bucket2.q, \
bucket3.q, \
bucket4.q, \
+ bucket_map_join_1.q, \
+ bucket_map_join_2.q, \
+ bucket_map_join_spark1.q, \
+ bucket_map_join_spark2.q, \
+ bucket_map_join_spark3.q, \
+ bucket_map_join_spark4.q, \
+ bucket_map_join_tez1.q, \
+ bucket_map_join_tez2.q, \
bucketmapjoin1.q, \
+ bucketmapjoin10.q, \
+ bucketmapjoin11.q, \
+ bucketmapjoin12.q, \
+ bucketmapjoin13.q, \
bucketmapjoin2.q, \
bucketmapjoin3.q, \
bucketmapjoin4.q, \
@@ -527,10 +539,6 @@ spark.query.files=add_part_multiple.q, \
bucketmapjoin7.q, \
bucketmapjoin8.q, \
bucketmapjoin9.q, \
- bucketmapjoin10.q, \
- bucketmapjoin11.q, \
- bucketmapjoin12.q, \
- bucketmapjoin13.q, \
bucketmapjoin_negative.q, \
bucketmapjoin_negative2.q, \
bucketmapjoin_negative3.q, \
@@ -539,14 +547,6 @@ spark.query.files=add_part_multiple.q, \
bucketsortoptimize_insert_6.q, \
bucketsortoptimize_insert_7.q, \
bucketsortoptimize_insert_8.q, \
- bucket_map_join_1.q, \
- bucket_map_join_2.q, \
- bucket_map_join_spark1.q \
- bucket_map_join_spark2.q \
- bucket_map_join_spark3.q \
- bucket_map_join_spark4.q \
- bucket_map_join_tez1.q, \
- bucket_map_join_tez2.q, \
column_access_stats.q, \
count.q, \
create_merge_compressed.q, \
@@ -567,6 +567,8 @@ spark.query.files=add_part_multiple.q, \
filter_join_breaktask.q, \
filter_join_breaktask2.q, \
groupby1.q, \
+ groupby10.q, \
+ groupby11.q, \
groupby2.q, \
groupby3.q, \
groupby3_map.q, \
@@ -586,8 +588,6 @@ spark.query.files=add_part_multiple.q, \
groupby8_map_skew.q, \
groupby8_noskew.q, \
groupby9.q, \
- groupby10.q, \
- groupby11.q, \
groupby_bigdata.q, \
groupby_complex_types.q, \
groupby_complex_types_multi_single_reducer.q, \
@@ -606,12 +606,12 @@ spark.query.files=add_part_multiple.q, \
having.q, \
index_auto_self_join.q, \
innerjoin.q, \
- input1_limit.q, \
input12.q, \
input13.q, \
input14.q, \
input17.q, \
input18.q, \
+ input1_limit.q, \
input_part2.q, \
insert1.q, \
insert_into1.q, \
@@ -619,14 +619,6 @@ spark.query.files=add_part_multiple.q, \
insert_into3.q, \
join0.q, \
join1.q, \
- join2.q, \
- join3.q, \
- join4.q, \
- join5.q, \
- join6.q, \
- join7.q, \
- join8.q, \
- join9.q, \
join10.q, \
join11.q, \
join12.q, \
@@ -638,6 +630,7 @@ spark.query.files=add_part_multiple.q, \
join18.q, \
join18_multi_distinct.q, \
join19.q, \
+ join2.q, \
join20.q, \
join21.q, \
join22.q, \
@@ -648,6 +641,7 @@ spark.query.files=add_part_multiple.q, \
join27.q, \
join28.q, \
join29.q, \
+ join3.q, \
join30.q, \
join31.q, \
join32.q, \
@@ -658,8 +652,14 @@ spark.query.files=add_part_multiple.q, \
join36.q, \
join37.q, \
join39.q, \
+ join4.q, \
join40.q, \
join41.q, \
+ join5.q, \
+ join6.q, \
+ join7.q, \
+ join8.q, \
+ join9.q, \
join_1to1.q, \
join_alt_syntax.q, \
join_array.q, \
@@ -672,7 +672,7 @@ spark.query.files=add_part_multiple.q, \
join_cond_pushdown_unqual2.q, \
join_cond_pushdown_unqual3.q, \
join_cond_pushdown_unqual4.q, \
- join_empty.q \
+ join_empty.q, \
join_filters_overlap.q, \
join_hive_626.q, \
join_map_ppr.q, \
@@ -693,6 +693,12 @@ spark.query.files=add_part_multiple.q, \
limit_pushdown.q, \
list_bucket_dml_2.q, \
load_dyn_part1.q, \
+ load_dyn_part10.q, \
+ load_dyn_part11.q, \
+ load_dyn_part12.q, \
+ load_dyn_part13.q, \
+ load_dyn_part14.q, \
+ load_dyn_part15.q, \
load_dyn_part2.q, \
load_dyn_part3.q, \
load_dyn_part4.q, \
@@ -701,12 +707,6 @@ spark.query.files=add_part_multiple.q, \
load_dyn_part7.q, \
load_dyn_part8.q, \
load_dyn_part9.q, \
- load_dyn_part10.q, \
- load_dyn_part11.q, \
- load_dyn_part12.q, \
- load_dyn_part13.q, \
- load_dyn_part14.q, \
- load_dyn_part15.q, \
louter_join_ppr.q, \
mapjoin1.q, \
mapjoin_addjar.q, \
@@ -726,7 +726,6 @@ spark.query.files=add_part_multiple.q, \
mergejoins_mixed.q, \
metadata_only_queries.q, \
metadata_only_queries_with_filters.q, \
- multigroupby_singlemr.q, \
multi_insert.q, \
multi_insert_gby.q, \
multi_insert_gby2.q, \
@@ -735,6 +734,7 @@ spark.query.files=add_part_multiple.q, \
multi_insert_mixed.q, \
multi_insert_move_tasks_share_dependencies.q, \
multi_join_union.q, \
+ multigroupby_singlemr.q, \
optimize_nullscan.q, \
order.q, \
order2.q, \
@@ -769,6 +769,7 @@ spark.query.files=add_part_multiple.q, \
reduce_deduplicate_exclude_join.q, \
router_join_ppr.q, \
sample1.q, \
+ sample10.q, \
sample2.q, \
sample3.q, \
sample4.q, \
@@ -777,22 +778,16 @@ spark.query.files=add_part_multiple.q, \
sample7.q, \
sample8.q, \
sample9.q, \
- sample10.q, \
- scriptfile1.q, \
script_env_var1.q, \
script_env_var2.q, \
script_pipe.q, \
+ scriptfile1.q, \
semijoin.q, \
skewjoin.q, \
+ skewjoin_noskew.q, \
+ skewjoin_union_remove_1.q, \
+ skewjoin_union_remove_2.q, \
skewjoinopt1.q, \
- skewjoinopt2.q, \
- skewjoinopt3.q, \
- skewjoinopt4.q, \
- skewjoinopt5.q, \
- skewjoinopt6.q, \
- skewjoinopt7.q, \
- skewjoinopt8.q, \
- skewjoinopt9.q, \
skewjoinopt10.q, \
skewjoinopt11.q, \
skewjoinopt12.q, \
@@ -803,24 +798,40 @@ spark.query.files=add_part_multiple.q, \
skewjoinopt17.q, \
skewjoinopt18.q, \
skewjoinopt19.q, \
+ skewjoinopt2.q, \
skewjoinopt20.q, \
- skewjoin_noskew.q, \
- skewjoin_union_remove_1.q, \
- skewjoin_union_remove_2.q, \
+ skewjoinopt3.q, \
+ skewjoinopt4.q, \
+ skewjoinopt5.q, \
+ skewjoinopt6.q, \
+ skewjoinopt7.q, \
+ skewjoinopt8.q, \
+ skewjoinopt9.q, \
+ smb_mapjoin_1.q, \
+ smb_mapjoin_10.q, \
+ smb_mapjoin_11.q, \
+ smb_mapjoin_12.q, \
+ smb_mapjoin_13.q, \
+ smb_mapjoin_14.q, \
+ smb_mapjoin_15.q, \
+ smb_mapjoin_16.q, \
+ smb_mapjoin_17.q, \
smb_mapjoin_18.q, \
smb_mapjoin_19.q, \
+ smb_mapjoin_2.q, \
smb_mapjoin_20.q, \
smb_mapjoin_21.q, \
smb_mapjoin_22.q, \
smb_mapjoin_25.q, \
+ smb_mapjoin_3.q, \
+ smb_mapjoin_4.q, \
+ smb_mapjoin_5.q, \
+ smb_mapjoin_6.q, \
+ smb_mapjoin_7.q, \
+ smb_mapjoin_8.q, \
+ smb_mapjoin_9.q, \
sort.q, \
spark_test.q, \
- stats_counter.q, \
- stats_counter_partitioned.q, \
- stats_noscan_1.q, \
- stats_noscan_2.q, \
- stats_only_null.q, \
- stats_partscan_1_23.q, \
stats0.q, \
stats1.q, \
stats10.q, \
@@ -838,14 +849,20 @@ spark.query.files=add_part_multiple.q, \
stats7.q, \
stats8.q, \
stats9.q, \
+ stats_counter.q, \
+ stats_counter_partitioned.q, \
+ stats_noscan_1.q, \
+ stats_noscan_2.q, \
+ stats_only_null.q, \
+ stats_partscan_1_23.q, \
statsfs.q, \
subquery_exists.q, \
subquery_multiinsert.q, \
table_access_keys_stats.q, \
temp_table.q, \
temp_table_join1.q, \
- tez_joins_explain.q, \
tez_join_tests.q, \
+ tez_joins_explain.q, \
timestamp_1.q, \
timestamp_2.q, \
timestamp_3.q, \
@@ -858,14 +875,6 @@ spark.query.files=add_part_multiple.q, \
transform_ppr1.q, \
transform_ppr2.q, \
union.q, \
- union2.q, \
- union3.q, \
- union4.q, \
- union5.q, \
- union6.q, \
- union7.q, \
- union8.q, \
- union9.q, \
union10.q, \
union11.q, \
union13.q, \
@@ -874,23 +883,23 @@ spark.query.files=add_part_multiple.q, \
union16.q, \
union18.q, \
union19.q, \
+ union2.q, \
union23.q, \
union25.q, \
union28.q, \
union29.q, \
+ union3.q, \
union30.q, \
union33.q, \
+ union4.q, \
+ union5.q, \
+ union6.q, \
+ union7.q, \
+ union8.q, \
+ union9.q, \
union_null.q, \
union_ppr.q, \
union_remove_1.q, \
- union_remove_2.q, \
- union_remove_3.q, \
- union_remove_4.q, \
- union_remove_5.q, \
- union_remove_6.q, \
- union_remove_7.q, \
- union_remove_8.q, \
- union_remove_9.q, \
union_remove_10.q, \
union_remove_11.q, \
union_remove_15.q, \
@@ -898,20 +907,37 @@ spark.query.files=add_part_multiple.q, \
union_remove_17.q, \
union_remove_18.q, \
union_remove_19.q, \
+ union_remove_2.q, \
union_remove_20.q, \
union_remove_21.q, \
union_remove_24.q, \
union_remove_25.q, \
+ union_remove_3.q, \
+ union_remove_4.q, \
+ union_remove_5.q, \
+ union_remove_6.q, \
+ union_remove_7.q, \
+ union_remove_8.q, \
+ union_remove_9.q, \
uniquejoin.q, \
varchar_join1.q, \
+ vector_between_in.q, \
+ vector_cast_constant.q, \
+ vector_char_4.q, \
+ vector_count_distinct.q, \
+ vector_data_types.q, \
+ vector_decimal_aggregate.q, \
+ vector_decimal_mapjoin.q, \
+ vector_distinct_2.q, \
+ vector_elt.q, \
+ vector_groupby_3.q, \
+ vector_left_outer_join.q, \
+ vector_mapjoin_reduce.q, \
+ vector_orderby_5.q, \
+ vector_string_concat.q, \
+ vector_varchar_4.q, \
vectorization_0.q, \
vectorization_1.q, \
- vectorization_2.q, \
- vectorization_3.q, \
- vectorization_4.q, \
- vectorization_5.q, \
- vectorization_6.q, \
- vectorization_9.q, \
vectorization_10.q, \
vectorization_11.q, \
vectorization_12.q, \
@@ -919,6 +945,12 @@ spark.query.files=add_part_multiple.q, \
vectorization_14.q, \
vectorization_15.q, \
vectorization_16.q, \
+ vectorization_2.q, \
+ vectorization_3.q, \
+ vectorization_4.q, \
+ vectorization_5.q, \
+ vectorization_6.q, \
+ vectorization_9.q, \
vectorization_decimal_date.q, \
vectorization_div0.q, \
vectorization_nested_udf.q, \
@@ -936,19 +968,4 @@ spark.query.files=add_part_multiple.q, \
vectorized_shufflejoin.q, \
vectorized_string_funcs.q, \
vectorized_timestamp_funcs.q, \
- vector_between_in.q, \
- vector_cast_constant.q, \
- vector_char_4.q, \
- vector_count_distinct.q, \
- vector_data_types.q, \
- vector_decimal_aggregate.q, \
- vector_decimal_mapjoin.q, \
- vector_distinct_2.q, \
- vector_elt.q, \
- vector_groupby_3.q, \
- vector_left_outer_join.q, \
- vector_mapjoin_reduce.q, \
- vector_orderby_5.q, \
- vector_string_concat.q, \
- vector_varchar_4.q, \
- windowing.q.q
+ windowing.q
Added:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkJoinHintOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkJoinHintOptimizer.java?rev=1646998&view=auto
==============================================================================
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkJoinHintOptimizer.java
(added)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkJoinHintOptimizer.java
Sat Dec 20 14:55:13 2014
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.spark;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.optimizer.BucketJoinProcCtx;
+import org.apache.hadoop.hive.ql.optimizer.BucketMapjoinProc;
+import org.apache.hadoop.hive.ql.optimizer.SortBucketJoinProcCtx;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.spark.OptimizeSparkProcContext;
+
+import java.util.Stack;
+
+/**
+ * This processes joins in which user specified a hint to identify the
small-table.
+ * Currently it takes a mapjoin already converted from hints, and converts it
further to BucketMapJoin or SMBMapJoin
+ * using same small-table identification.
+ *
+ * The idea is eventually to process even hinted Mapjoin hints here, but due
to code complexity in refactoring, that is still
+ * in Optimizer.
+ */
+public class SparkJoinHintOptimizer implements NodeProcessor {
+
+ private BucketMapjoinProc bucketMapJoinOptimizer;
+ private SparkSMBJoinHintOptimizer smbMapJoinOptimizer;
+
+ public SparkJoinHintOptimizer(ParseContext parseCtx) {
+ bucketMapJoinOptimizer = new BucketMapjoinProc(parseCtx);
+ smbMapJoinOptimizer = new SparkSMBJoinHintOptimizer(parseCtx);
+ }
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
+ MapJoinOperator mapJoinOp = (MapJoinOperator) nd;
+ OptimizeSparkProcContext context = (OptimizeSparkProcContext) procCtx;
+ HiveConf hiveConf = context.getParseContext().getConf();
+
+ // Convert from mapjoin to bucket map join if enabled.
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEOPTBUCKETMAPJOIN) ||
hiveConf.getBoolVar(HiveConf.ConfVars.HIVEOPTSORTMERGEBUCKETMAPJOIN)) {
+ BucketJoinProcCtx bjProcCtx = new BucketJoinProcCtx(hiveConf);
+ bucketMapJoinOptimizer.process(nd, stack, bjProcCtx, nodeOutputs);
+ }
+
+ // Convert from bucket map join to sort merge bucket map join if enabled.
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEOPTSORTMERGEBUCKETMAPJOIN)) {
+ SortBucketJoinProcCtx smbJoinCtx = new SortBucketJoinProcCtx(hiveConf);
+ smbMapJoinOptimizer.process(nd, stack, smbJoinCtx, nodeOutputs);
+ }
+ return null;
+ }
+}
\ No newline at end of file
Added:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java?rev=1646998&view=auto
==============================================================================
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java
(added)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java
Sat Dec 20 14:55:13 2014
@@ -0,0 +1,83 @@
+package org.apache.hadoop.hive.ql.optimizer.spark;
+
+import com.clearspring.analytics.util.Preconditions;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.optimizer.AbstractSMBJoinProc;
+import org.apache.hadoop.hive.ql.optimizer.SortBucketJoinProcCtx;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
+
+import java.util.List;
+import java.util.Stack;
+
+/**
+ * Converts from a bucket-mapjoin created from hints to SMB mapjoin.
+ */
+public class SparkSMBJoinHintOptimizer extends AbstractSMBJoinProc implements
NodeProcessor {
+
+ public SparkSMBJoinHintOptimizer(ParseContext pctx) {
+ super(pctx);
+ }
+
+ public SparkSMBJoinHintOptimizer() {
+ }
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ MapJoinOperator mapJoinOp = (MapJoinOperator) nd;
+ SortBucketJoinProcCtx smbJoinContext = (SortBucketJoinProcCtx) procCtx;
+
+ boolean convert =
+ canConvertBucketMapJoinToSMBJoin(mapJoinOp, stack, smbJoinContext,
nodeOutputs);
+
+ // Throw an error if the user asked for sort merge bucketed mapjoin to be
enforced
+ // and sort merge bucketed mapjoin cannot be performed
+ if (!convert &&
+ pGraphContext.getConf().getBoolVar(
+ HiveConf.ConfVars.HIVEENFORCESORTMERGEBUCKETMAPJOIN)) {
+ throw new SemanticException(ErrorMsg.SORTMERGE_MAPJOIN_FAILED.getMsg());
+ }
+
+ if (convert) {
+ removeSmallTableReduceSink(mapJoinOp);
+ convertBucketMapJoinToSMBJoin(mapJoinOp, smbJoinContext, pGraphContext);
+ }
+ return null;
+ }
+
+ /**
+ * In bucket mapjoin, there are ReduceSinks that mark a small table parent
(Reduce Sink are removed from big-table).
+ * In SMB join these are not expected for any parents, either from small or
big tables.
+ * @param mapJoinOp
+ */
+ private void removeSmallTableReduceSink(MapJoinOperator mapJoinOp) {
+ SMBJoinDesc smbJoinDesc = new SMBJoinDesc(mapJoinOp.getConf());
+ List<Operator<? extends OperatorDesc>> parentOperators =
mapJoinOp.getParentOperators();
+ for (int i = 0; i < parentOperators.size(); i++) {
+ Operator<? extends OperatorDesc> par = parentOperators.get(i);
+ if (i != smbJoinDesc.getPosBigTable()) {
+ if (par instanceof ReduceSinkOperator) {
+ List<Operator<? extends OperatorDesc>> grandParents =
par.getParentOperators();
+ Preconditions.checkArgument(grandParents.size() == 1,
+ "AssertionError: expect # of parents to be 1, but was " +
grandParents.size());
+ Operator<? extends OperatorDesc> grandParent = grandParents.get(0);
+ grandParent.removeChild(par);
+ grandParent.setChildOperators(Utilities.makeList(mapJoinOp));
+ mapJoinOp.getParentOperators().set(i, grandParent);
+ }
+ }
+ }
+ }
+}
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSortMergeJoinOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSortMergeJoinOptimizer.java?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSortMergeJoinOptimizer.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSortMergeJoinOptimizer.java
Sat Dec 20 14:55:13 2014
@@ -36,7 +36,7 @@ import org.apache.hadoop.hive.ql.plan.Op
import java.util.Stack;
/**
- * Converts a join operator to an SMB join if eligible.
+ * Converts a common join operator to an SMB join if eligible. Handles auto
SMB conversion.
*/
public class SparkSortMergeJoinOptimizer extends AbstractSMBJoinProc
implements NodeProcessor {
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
Sat Dec 20 14:55:13 2014
@@ -67,6 +67,7 @@ import org.apache.hadoop.hive.ql.optimiz
import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer;
import org.apache.hadoop.hive.ql.optimizer.spark.SetSparkReducerParallelism;
+import org.apache.hadoop.hive.ql.optimizer.spark.SparkJoinHintOptimizer;
import org.apache.hadoop.hive.ql.optimizer.spark.SparkJoinOptimizer;
import org.apache.hadoop.hive.ql.optimizer.spark.SparkReduceSinkMapJoinProc;
import org.apache.hadoop.hive.ql.optimizer.spark.SparkSkewJoinResolver;
@@ -123,8 +124,9 @@ public class SparkCompiler extends TaskC
ReduceSinkOperator.getOperatorName() + "%"),
new SetSparkReducerParallelism());
- opRules.put(new RuleRegExp(new String("Convert Join to Map-join"),
- JoinOperator.getOperatorName() + "%"), new SparkJoinOptimizer(pCtx));
+ opRules.put(new TypeRule(JoinOperator.class), new
SparkJoinOptimizer(pCtx));
+
+ opRules.put(new TypeRule(MapJoinOperator.class), new
SparkJoinHintOptimizer(pCtx));
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
Sat Dec 20 14:55:13 2014
@@ -103,6 +103,9 @@ STAGE PLANS:
Map 2
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+ b {}
Stage: Stage-1
Spark
@@ -111,6 +114,9 @@ STAGE PLANS:
Map 1
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+ b {}
Stage: Stage-0
Fetch Operator
@@ -211,6 +217,9 @@ STAGE PLANS:
Map 1
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+ a {}
Stage: Stage-1
Spark
@@ -219,6 +228,9 @@ STAGE PLANS:
Map 2
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+ a {}
Stage: Stage-0
Fetch Operator
@@ -430,6 +442,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -511,6 +528,7 @@ STAGE PLANS:
1 Map 2
Position of Big Table: 0
Statistics: Num rows: 30 Data size: 3253 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col6 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -543,6 +561,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -818,6 +841,11 @@ STAGE PLANS:
Position of Big Table: 1
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -894,6 +922,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 30 Data size: 3253 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col6 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -931,6 +960,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out
Sat Dec 20 14:55:13 2014
@@ -231,6 +231,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -361,6 +366,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 1177 Data size: 4709 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -374,6 +380,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -638,6 +649,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -768,6 +784,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 1177 Data size: 4709 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -781,6 +798,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out
Sat Dec 20 14:55:13 2014
@@ -190,6 +190,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -271,6 +276,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 378 Data size: 1514 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -284,6 +290,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out
Sat Dec 20 14:55:13 2014
@@ -493,6 +493,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -575,6 +580,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -588,6 +594,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -788,6 +799,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -870,6 +886,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -883,6 +900,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1085,6 +1107,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1167,6 +1194,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -1180,6 +1208,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out
Sat Dec 20 14:55:13 2014
@@ -188,6 +188,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -269,6 +274,7 @@ STAGE PLANS:
1 Map 2
Position of Big Table: 0
Statistics: Num rows: 30 Data size: 3253 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col7 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -301,6 +307,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -585,6 +596,11 @@ STAGE PLANS:
Position of Big Table: 1
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -666,6 +682,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 30 Data size: 3253 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col7 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -703,6 +720,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1004,6 +1026,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1133,6 +1160,7 @@ STAGE PLANS:
1 Map 2
Position of Big Table: 0
Statistics: Num rows: 31 Data size: 3368 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col7 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -1170,6 +1198,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out
Sat Dec 20 14:55:13 2014
@@ -219,6 +219,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -300,6 +305,7 @@ STAGE PLANS:
1 Map 2
Position of Big Table: 0
Statistics: Num rows: 30 Data size: 3253 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col7 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -332,6 +338,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -623,6 +634,11 @@ STAGE PLANS:
Position of Big Table: 1
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -704,6 +720,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 30 Data size: 3253 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col7 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -741,6 +758,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out
Sat Dec 20 14:55:13 2014
@@ -205,6 +205,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -281,6 +286,7 @@ STAGE PLANS:
1 Map 2
Position of Big Table: 0
Statistics: Num rows: 14 Data size: 1512 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col6 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -313,6 +319,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -573,6 +584,11 @@ STAGE PLANS:
Position of Big Table: 1
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -649,6 +665,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 14 Data size: 1512 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col6 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -686,6 +703,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out
Sat Dec 20 14:55:13 2014
@@ -255,6 +255,11 @@ STAGE PLANS:
Position of Big Table: 1
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -331,6 +336,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 60 Data size: 6393 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col6 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -363,6 +369,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -688,6 +699,11 @@ STAGE PLANS:
Position of Big Table: 1
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -764,6 +780,7 @@ STAGE PLANS:
0 Map 1
Position of Big Table: 1
Statistics: Num rows: 31 Data size: 3368 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col6 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -801,6 +818,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out
Sat Dec 20 14:55:13 2014
@@ -169,6 +169,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -253,6 +258,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 378 Data size: 1514 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col8 (type: string)
outputColumnNames: _col0, _col1
@@ -265,6 +271,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out
Sat Dec 20 14:55:13 2014
@@ -155,6 +155,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -237,6 +242,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 378 Data size: 1514 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -250,6 +256,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -464,6 +475,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -546,6 +562,7 @@ STAGE PLANS:
1 Map 3
Position of Big Table: 0
Statistics: Num rows: 378 Data size: 1514 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Group By Operator
aggregations: count()
mode: hash
@@ -559,6 +576,11 @@ STAGE PLANS:
auto parallelism: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out
Sat Dec 20 14:55:13 2014
@@ -165,6 +165,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -294,6 +299,7 @@ STAGE PLANS:
1 Map 2
Position of Big Table: 0
Statistics: Num rows: 31 Data size: 3368 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string),
_col6 (type: string)
outputColumnNames: _col0, _col1, _col2
@@ -326,6 +332,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out
Sat Dec 20 14:55:13 2014
@@ -224,6 +224,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -302,6 +307,7 @@ STAGE PLANS:
1 Map 2
Position of Big Table: 0
Statistics: Num rows: 6 Data size: 1320 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: string), _col1 (type:
string), _col5 (type: string), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -329,6 +335,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -465,6 +476,11 @@ STAGE PLANS:
Position of Big Table: 0
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -543,6 +559,7 @@ STAGE PLANS:
1 Map 2
Position of Big Table: 0
Statistics: Num rows: 6 Data size: 1320 Basic stats:
COMPLETE Column stats: NONE
+ BucketMapJoin: true
Select Operator
expressions: _col0 (type: string), _col1 (type:
string), _col5 (type: string), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -570,6 +587,11 @@ STAGE PLANS:
MultiFileSpray: false
Local Work:
Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out?rev=1646998&r1=1646997&r2=1646998&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out
Sat Dec 20 14:55:13 2014
@@ -112,48 +112,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-1
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL
SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
- Map 3
- Reducer 2
- Needs Tagging: true
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- outputColumnNames: _col0, _col6, _col7
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column
stats: NONE
- Select Operator
- expressions: _col6 (type: int), _col7 (type: string),
'2010-10-15' (type: string), _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE
Column stats: NONE
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE
Column stats: NONE
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- columns _col0,_col1,_col2,_col3
- columns.types int:string:string:int
- escape.delim \
- hive.serialization.extend.nesting.levels true
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
Stage: Stage-0
Fetch Operator
@@ -273,48 +234,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-1
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL
SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
- Map 3
- Reducer 2
- Needs Tagging: true
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- outputColumnNames: _col0, _col6, _col7
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column
stats: NONE
- Select Operator
- expressions: _col6 (type: int), _col7 (type: string),
'2010-10-15' (type: string), _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE
Column stats: NONE
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE
Column stats: NONE
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- columns _col0,_col1,_col2,_col3
- columns.types int:string:string:int
- escape.delim \
- hive.serialization.extend.nesting.levels true
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
Stage: Stage-0
Fetch Operator
@@ -381,8 +303,6 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-1
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL
SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -393,47 +313,29 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
- Map 3
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- outputColumnNames: _col0, _col6, _col7
- Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
- Select Operator
- expressions: _col6 (type: int), _col7 (type: string),
'2010-10-15' (type: string), _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.smb_mapjoin9_results
+ Sorted Merge Bucket Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1 {key} {value}
+ keys:
+ 0 key (type: int)
+ 1 key (type: int)
+ outputColumnNames: _col0, _col6, _col7
+ Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col6 (type: int), _col7 (type: string),
'2010-10-15' (type: string), _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 275 Data size: 2921 Basic
stats: COMPLETE Column stats: NONE
+ table:
+ input format:
org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.smb_mapjoin9_results
Stage: Stage-0
Move Operator
