Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out Thu Dec 18 19:58:10 2014 @@ -95,12 +95,13 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key WHERE a.ds = '1' and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -112,7 +113,31 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -122,16 +147,33 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col7 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col7) (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -229,15 +271,14 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key WHERE a.ds is not null and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -248,7 +289,31 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -258,19 +323,28 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col7 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col7) (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) - Reducer 2 + Local Work: + Map Reduce Local Work + Reducer 3 Reduce Operator Tree: Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -397,12 +471,13 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key WHERE a.ds = '1' and b.ds is not null POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -414,7 +489,31 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 168 Data size: 1472 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -424,16 +523,33 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col7 + input vertices: + 0 Map 1 + Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col7) (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -559,12 +675,13 @@ JOIN ON a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -580,7 +697,35 @@ STAGE PLANS: expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: test_table2 + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -590,16 +735,33 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -707,12 +869,13 @@ JOIN ON a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -728,7 +891,35 @@ STAGE PLANS: expressions: key (type: int), concat(value, value) (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: test_table2 + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), concat(value, value) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -738,16 +929,33 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -855,15 +1063,14 @@ JOIN ON a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -878,7 +1085,35 @@ STAGE PLANS: expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: test_table2 + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -888,19 +1123,28 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) - Reducer 2 + Local Work: + Map Reduce Local Work + Reducer 3 Reduce Operator Tree: Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out Thu Dec 18 19:58:10 2014 @@ -71,12 +71,13 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key WHERE a.ds = '1' and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -88,7 +89,31 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -98,16 +123,33 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col7 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col0 (type: int), concat(_col1, _col7) (type: string) outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -224,15 +266,14 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key WHERE a.ds = '1' and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -243,7 +284,31 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -253,16 +318,24 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) - Reducer 2 + Local Work: + Map Reduce Local Work + Reducer 3 Reduce Operator Tree: Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out Thu Dec 18 19:58:10 2014 @@ -73,12 +73,13 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key and a.key2 = b.key2 WHERE a.ds = '1' and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -90,7 +91,31 @@ STAGE PLANS: Filter Operator predicate: (key is not null and key2 is not null) (type: boolean) Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -100,16 +125,33 @@ STAGE PLANS: 0 key (type: int), key2 (type: int) 1 key (type: int), key2 (type: int) outputColumnNames: _col0, _col1, _col2, _col9 + input vertices: + 0 Map 1 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string) outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -214,12 +256,13 @@ ON a.key = b.key and a.key2 = b.key2 WHE )subq1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -231,7 +274,31 @@ STAGE PLANS: Filter Operator predicate: (key is not null and key2 is not null) (type: boolean) Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -241,16 +308,33 @@ STAGE PLANS: 0 key (type: int), key2 (type: int) 1 key (type: int), key2 (type: int) outputColumnNames: _col0, _col1, _col2, _col9 + input vertices: + 0 Map 1 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string) outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -355,15 +439,14 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key and a.key2 = b.key2 WHERE a.ds = '1' and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -374,7 +457,31 @@ STAGE PLANS: Filter Operator predicate: (key is not null and key2 is not null) (type: boolean) Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -384,19 +491,28 @@ STAGE PLANS: 0 key (type: int), key2 (type: int) 1 key (type: int), key2 (type: int) outputColumnNames: _col0, _col1, _col2, _col9 + input vertices: + 0 Map 1 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col9) (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Reducer 2 + Local Work: + Map Reduce Local Work + Reducer 3 Reduce Operator Tree: Extract + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -441,15 +557,14 @@ ON a.key = b.key and a.key2 = b.key2 WHE )subq1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -460,7 +575,31 @@ STAGE PLANS: Filter Operator predicate: (key is not null and key2 is not null) (type: boolean) Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -470,19 +609,28 @@ STAGE PLANS: 0 key (type: int), key2 (type: int) 1 key (type: int), key2 (type: int) outputColumnNames: _col0, _col1, _col2, _col9 + input vertices: + 0 Map 1 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col9) (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Reducer 2 + Local Work: + Map Reduce Local Work + Reducer 3 Reduce Operator Tree: Extract + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -533,12 +681,13 @@ ON a.key = b.key and a.key2 = b.key2 WHE )subq2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -550,7 +699,31 @@ STAGE PLANS: Filter Operator predicate: (key is not null and key2 is not null) (type: boolean) Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -560,16 +733,33 @@ STAGE PLANS: 0 key (type: int), key2 (type: int) 1 key (type: int), key2 (type: int) outputColumnNames: _col0, _col1, _col2, _col9 + input vertices: + 0 Map 1 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string) outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -692,12 +882,13 @@ ON a.key = b.key and a.key2 = b.key2 WHE )subq2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -709,7 +900,31 @@ STAGE PLANS: Filter Operator predicate: (key is not null and key2 is not null) (type: boolean) Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -719,16 +934,33 @@ STAGE PLANS: 0 key (type: int), key2 (type: int) 1 key (type: int), key2 (type: int) outputColumnNames: _col0, _col1, _col2, _col9 + input vertices: + 0 Map 1 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string) outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -861,15 +1093,14 @@ ON a.key = b.key and a.key2 = b.key2 WHE )subq2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -880,7 +1111,31 @@ STAGE PLANS: Filter Operator predicate: (key is not null and key2 is not null) (type: boolean) Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -890,19 +1145,28 @@ STAGE PLANS: 0 key (type: int), key2 (type: int) 1 key (type: int), key2 (type: int) outputColumnNames: _col0, _col1, _col2, _col9 + input vertices: + 0 Map 1 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: -- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Reducer 2 + Local Work: + Map Reduce Local Work + Reducer 3 Reduce Operator Tree: Extract + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out Thu Dec 18 19:58:10 2014 @@ -73,12 +73,13 @@ ON a.key = b.key WHERE a.ds = '1' and b. and (a.key = 0 or a.key = 5) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -90,7 +91,31 @@ STAGE PLANS: Filter Operator predicate: (key is not null and ((key = 0) or (key = 5))) (type: boolean) Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and ((key = 0) or (key = 5))) (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -100,16 +125,33 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col7 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col7) (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -209,12 +251,13 @@ JOIN ON a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -230,7 +273,35 @@ STAGE PLANS: expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: test_table2 + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key = 0) or (key = 5)) and key is not null) (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -240,16 +311,33 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -355,12 +443,13 @@ ON a.key = b.key WHERE a.key = 0 or a.key = 5 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -376,7 +465,35 @@ STAGE PLANS: expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: test_table2 + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key < 8) and key is not null) and ((key = 0) or (key = 5))) (type: boolean) + Statistics: Num rows: 14 Data size: 122 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 122 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -386,16 +503,33 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 + input vertices: + 0 Map 1 + Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out Thu Dec 18 19:58:10 2014 @@ -71,12 +71,13 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key WHERE a.ds = '1' and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -88,7 +89,31 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -98,16 +123,33 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col6, _col7 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col6 (type: int), concat(_col1, _col7) (type: string) outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -206,12 +248,13 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key WHERE a.ds = '1' and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -223,7 +266,31 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Spark HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator condition map: Inner Join 0 to 1 condition expressions: @@ -233,16 +300,33 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col6, _col7 + input vertices: + 0 Map 1 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col6 (type: int), _col0 (type: int), concat(_col1, _col7) (type: string) outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Local Work: + Map Reduce Local Work + Reducer 3 + Reduce Operator Tree: + Extract + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out Thu Dec 18 19:58:10 2014 @@ -96,7 +96,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[16][bigTable=a] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[15][bigTable=a] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -207,7 +207,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[22][bigTable=a] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=a] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -345,8 +345,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[18][bigTable=a] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[17][bigTable=d1] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[17][bigTable=a] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=d1] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -472,7 +472,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/parquet_join.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/parquet_join.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/parquet_join.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/parquet_join.q.out Thu Dec 18 19:58:10 2014 @@ -319,11 +319,14 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col1, _col7 + Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col7 (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
