http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out new file mode 100644 index 0000000..2b13dc6 --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out @@ -0,0 +1,1406 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +create table sorted_mod_4 stored as orc +as select ctinyint, pmod(cint, 4) as cmodint from alltypesorc +where cint is not null and ctinyint is not null +order by ctinyint +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@sorted_mod_4 +POSTHOOK: query: -- SORT_QUERY_RESULTS + +create table sorted_mod_4 stored as orc +as select ctinyint, pmod(cint, 4) as cmodint from alltypesorc +where cint is not null and ctinyint is not null +order by ctinyint +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sorted_mod_4 +PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@sorted_mod_4 +PREHOOK: Output: default@sorted_mod_4 +POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sorted_mod_4 +POSTHOOK: Output: default@sorted_mod_4 +PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +PREHOOK: query: create table small_table stored +as orc as select ctinyint, cbigint from alltypesorc limit 100 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_table +POSTHOOK: query: create table small_table stored +as orc as select ctinyint, cbigint from alltypesorc limit 100 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_table +PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Output: default@small_table +POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Output: default@small_table +PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +#### A masked pattern was here #### +PREHOOK: query: explain +select count(*) from (select s.*, st.* +from sorted_mod_4 s +left outer join small_table st +on s.ctinyint = st.ctinyint +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.*, st.* +from sorted_mod_4 s +left outer join small_table st +on s.ctinyint = st.ctinyint +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: st + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 2229 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.*, st.* +from sorted_mod_4 s +left outer join small_table st +on s.ctinyint = st.ctinyint +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.*, st.* +from sorted_mod_4 s +left outer join small_table st +on s.ctinyint = st.ctinyint +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +6876 +PREHOOK: query: explain +select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and s.cmodint = 2 +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and s.cmodint = 2 +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 {(_col1 = 2)} + 1 + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), cmodint (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + filter predicates: + 0 {(_col1 = 2)} + 1 + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 2229 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and s.cmodint = 2 +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and s.cmodint = 2 +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +6058 +PREHOOK: query: explain +select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 {((UDFToInteger(_col0) pmod 4) = _col1)} + 1 + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), cmodint (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + filter predicates: + 0 {((UDFToInteger(_col0) pmod 4) = _col1)} + 1 + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 2229 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +6248 +PREHOOK: query: explain +select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and s.ctinyint < 100 +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and s.ctinyint < 100 +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 {(_col0 < 100)} + 1 + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + filter predicates: + 0 {(_col0 < 100)} + 1 + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 2229 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and s.ctinyint < 100 +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint +from sorted_mod_4 s +left outer join small_table sm +on s.ctinyint = sm.ctinyint and s.ctinyint < 100 +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +6876 +PREHOOK: query: explain +select count(*) from (select s.*, sm.*, s2.* +from sorted_mod_4 s +left outer join small_table sm + on pmod(sm.cbigint, 8) = s.cmodint +left outer join sorted_mod_4 s2 + on s2.ctinyint = s.ctinyint +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.*, sm.*, s2.* +from sorted_mod_4 s +left outer join small_table sm + on pmod(sm.cbigint, 8) = s.cmodint +left outer join sorted_mod_4 s2 + on s2.ctinyint = s.ctinyint +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 UDFToLong(_col1) (type: bigint) + 1 (_col0 pmod UDFToLong(8)) (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Map 4 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), cmodint (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6058 Data size: 2027 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 UDFToLong(_col1) (type: bigint) + 1 (_col0 pmod UDFToLong(8)) (type: bigint) + outputColumnNames: _col0 + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 2229 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) + input vertices: + 1 Map 4 + Statistics: Num rows: 7329 Data size: 2451 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.*, sm.*, s2.* +from sorted_mod_4 s +left outer join small_table sm + on pmod(sm.cbigint, 8) = s.cmodint +left outer join sorted_mod_4 s2 + on s2.ctinyint = s.ctinyint +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table +PREHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.*, sm.*, s2.* +from sorted_mod_4 s +left outer join small_table sm + on pmod(sm.cbigint, 8) = s.cmodint +left outer join sorted_mod_4 s2 + on s2.ctinyint = s.ctinyint +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table +POSTHOOK: Input: default@sorted_mod_4 +#### A masked pattern was here #### +3268334 +PREHOOK: query: create table mod_8_mod_4 stored as orc +as select pmod(ctinyint, 8) as cmodtinyint, pmod(cint, 4) as cmodint from alltypesorc +where cint is not null and ctinyint is not null +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@mod_8_mod_4 +POSTHOOK: query: create table mod_8_mod_4 stored as orc +as select pmod(ctinyint, 8) as cmodtinyint, pmod(cint, 4) as cmodint from alltypesorc +where cint is not null and ctinyint is not null +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mod_8_mod_4 +PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Output: default@mod_8_mod_4 +POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Output: default@mod_8_mod_4 +PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@mod_8_mod_4 +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mod_8_mod_4 +#### A masked pattern was here #### +PREHOOK: query: create table small_table2 stored +as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@small_table2 +POSTHOOK: query: create table small_table2 stored +as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_table2 +PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table2 +PREHOOK: Output: default@small_table2 +POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table2 +POSTHOOK: Output: default@small_table2 +PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@small_table2 +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@small_table2 +#### A masked pattern was here #### +PREHOOK: query: explain +select count(*) from (select s.*, st.* +from mod_8_mod_4 s +left outer join small_table2 st +on s.cmodtinyint = st.cmodtinyint +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.*, st.* +from mod_8_mod_4 s +left outer join small_table2 st +on s.cmodtinyint = st.cmodtinyint +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: st + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 3072 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.*, st.* +from mod_8_mod_4 s +left outer join small_table2 st +on s.cmodtinyint = st.cmodtinyint +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Input: default@small_table2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.*, st.* +from mod_8_mod_4 s +left outer join small_table2 st +on s.cmodtinyint = st.cmodtinyint +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Input: default@small_table2 +#### A masked pattern was here #### +39112 +PREHOOK: query: explain +select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2 +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2 +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 {(_col1 = 2)} + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int), cmodint (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + filter predicates: + 0 {(_col1 = 2)} + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 3072 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2 +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Input: default@small_table2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2 +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Input: default@small_table2 +#### A masked pattern was here #### +11171 +PREHOOK: query: explain +select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 {((_col0 pmod 4) = _col1)} + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int), cmodint (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + filter predicates: + 0 {((_col0 pmod 4) = _col1)} + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 3072 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Input: default@small_table2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Input: default@small_table2 +#### A masked pattern was here #### +14371 +PREHOOK: query: explain +select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3 +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3 +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 {(_col0 < 3)} + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + filter predicates: + 0 {(_col0 < 3)} + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 3072 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3 +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Input: default@small_table2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint +from mod_8_mod_4 s +left outer join small_table2 sm +on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3 +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Input: default@small_table2 +#### A masked pattern was here #### +17792 +PREHOOK: query: explain +select count(*) from (select s.*, sm.*, s2.* +from mod_8_mod_4 s +left outer join small_table2 sm + on pmod(sm.cbigint, 8) = s.cmodint +left outer join mod_8_mod_4 s2 + on s2.cmodtinyint = s.cmodtinyint +) t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from (select s.*, sm.*, s2.* +from mod_8_mod_4 s +left outer join small_table2 sm + on pmod(sm.cbigint, 8) = s.cmodint +left outer join mod_8_mod_4 s2 + on s2.cmodtinyint = s.cmodtinyint +) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 UDFToLong(_col1) (type: bigint) + 1 (_col0 pmod UDFToLong(8)) (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Map 4 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cmodtinyint (type: int), cmodint (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6058 Data size: 2793 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 UDFToLong(_col1) (type: bigint) + 1 (_col0 pmod UDFToLong(8)) (type: bigint) + outputColumnNames: _col0 + input vertices: + 1 Map 3 + Statistics: Num rows: 6663 Data size: 3072 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + input vertices: + 1 Map 4 + Statistics: Num rows: 7329 Data size: 3379 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Local Work: + Map Reduce Local Work + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from (select s.*, sm.*, s2.* +from mod_8_mod_4 s +left outer join small_table2 sm + on pmod(sm.cbigint, 8) = s.cmodint +left outer join mod_8_mod_4 s2 + on s2.cmodtinyint = s.cmodtinyint +) t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Input: default@small_table2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from (select s.*, sm.*, s2.* +from mod_8_mod_4 s +left outer join small_table2 sm + on pmod(sm.cbigint, 8) = s.cmodint +left outer join mod_8_mod_4 s2 + on s2.cmodtinyint = s.cmodtinyint +) t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Input: default@small_table2 +#### A masked pattern was here #### +6524438
http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/ql/src/test/templates/TestCliDriver.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm index 01745da..fa638ae 100644 --- a/ql/src/test/templates/TestCliDriver.vm +++ b/ql/src/test/templates/TestCliDriver.vm @@ -17,28 +17,21 @@ */ package org.apache.hadoop.hive.cli; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -import java.io.*; -import java.util.*; - import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; -import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -public class $className extends TestCase { +public class $className { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; - public static class TestCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { - public void addTestToSuite(TestSuite suite, Object setup, String tName) { - suite.addTest(new $className("testCliDriver_"+tName)); - } - } - static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); @@ -66,12 +59,8 @@ public class $className extends TestCase { } } - public $className(String name) { - super(name); - } - - @Override - protected void setUp() { + @Before + public void setUp() { try { qt.clearTestSideEffects(); } catch (Exception e) { @@ -82,21 +71,11 @@ public class $className extends TestCase { } } - /** - * Dummy last test. This is only meant to shutdown qt - */ - public void testCliDriver_shutdown() { - System.err.println ("Cleaning up " + "$className"); - } - - @Override - protected void tearDown() { + @After + public void tearDown() { try { qt.clearPostTestEffects(); - if (getName().equals("testCliDriver_shutdown")) - qt.shutdown(); - } - catch (Exception e) { + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); @@ -104,24 +83,16 @@ public class $className extends TestCase { } } - public static Test suite() { - Set<String> qFilesToExecute = new HashSet<String>(); - String qFiles = System.getProperty("qfile", "").trim(); - if(!qFiles.isEmpty()) { - for(String qFile : qFiles.split(",")) { - qFile = qFile.trim(); - if(!qFile.isEmpty()) { - qFilesToExecute.add(qFile); - } - } + @AfterClass + public static void shutdown() throws Exception { + try { + qt.shutdown(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in shutdown"); } - - TestSuite suite = new TestSuite(); - - QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, - suite, null, new TestCliDriverAddTestFromQFiles()); - suite.addTest(new $className("testCliDriver_shutdown")); - return suite; } static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " @@ -132,6 +103,7 @@ public class $className extends TestCase { #set ($eidx = $fname.indexOf('.')) #set ($tname = $fname.substring(0, $eidx)) #set ($fpath = $qfilesMap.get($fname)) + @Test public void testCliDriver_$tname() throws Exception { runTest("$tname", "$fname", (HIVE_ROOT + "$fpath")); } http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/ql/src/test/templates/TestCompareCliDriver.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestCompareCliDriver.vm b/ql/src/test/templates/TestCompareCliDriver.vm index 7f849e0..5e44315 100644 --- a/ql/src/test/templates/TestCompareCliDriver.vm +++ b/ql/src/test/templates/TestCompareCliDriver.vm @@ -17,29 +17,23 @@ */ package org.apache.hadoop.hive.cli; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - import java.io.*; import java.util.*; -import java.util.Arrays; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; -import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.fail; -public class $className extends TestCase { +public class $className { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; - public static class TestCompareCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { - public void addTestToSuite(TestSuite suite, Object setup, String tName) { - suite.addTest(new $className("testCompareCliDriver_"+tName)); - } - } - static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); @@ -66,12 +60,8 @@ public class $className extends TestCase { } } - public $className(String name) { - super(name); - } - - @Override - protected void setUp() { + @Before + public void setUp() { try { qt.clearTestSideEffects(); } catch (Exception e) { @@ -82,21 +72,11 @@ public class $className extends TestCase { } } - /** - * Dummy last test. This is only meant to shutdown qt - */ - public void testCompareCliDriver_shutdown() { - System.err.println ("Cleaning up " + "$className"); - } - - @Override - protected void tearDown() { + @After + public void tearDown() { try { qt.clearPostTestEffects(); - if (getName().equals("testCompareCliDriver_shutdown")) - qt.shutdown(); - } - catch (Exception e) { + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); @@ -104,24 +84,16 @@ public class $className extends TestCase { } } - public static Test suite() { - Set<String> qFilesToExecute = new HashSet<String>(); - String qFiles = System.getProperty("qfile", "").trim(); - if(!qFiles.isEmpty()) { - for(String qFile : qFiles.split(",")) { - qFile = qFile.trim(); - if(!qFile.isEmpty()) { - qFilesToExecute.add(qFile); - } - } + @AfterClass + public static void shutdown() throws Exception { + try { + qt.shutdown(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in shutdown"); } - - TestSuite suite = new TestSuite(); - - QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, - suite, null, new TestCompareCliDriverAddTestFromQFiles()); - suite.addTest(new $className("testCompareCliDriver_shutdown")); - return suite; } private Map<String, List<String>> versionFiles = new HashMap<String, List<String>>(); @@ -134,6 +106,7 @@ public class $className extends TestCase { #set ($eidx = $fname.indexOf('.')) #set ($tname = $fname.substring(0, $eidx)) #set ($fpath = $qfilesMap.get($fname)) + @Test public void testCompareCliDriver_$tname() throws Exception { runTest("$tname", "$fname", (HIVE_ROOT + "$fpath")); } http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/ql/src/test/templates/TestNegativeCliDriver.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestNegativeCliDriver.vm b/ql/src/test/templates/TestNegativeCliDriver.vm index 5f8ee8e..85c1e7f 100644 --- a/ql/src/test/templates/TestNegativeCliDriver.vm +++ b/ql/src/test/templates/TestNegativeCliDriver.vm @@ -17,28 +17,21 @@ */ package org.apache.hadoop.hive.cli; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -import java.io.*; -import java.util.*; - import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; -import org.apache.hadoop.hive.ql.exec.Task; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -public class $className extends TestCase { +public class $className { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; - public static class TestNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { - public void addTestToSuite(TestSuite suite, Object setup, String tName) { - suite.addTest(new $className("testNegativeCliDriver_"+tName)); - } - } - static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); String initScript = "$initScript"; @@ -59,30 +52,22 @@ public class $className extends TestCase { } } - public $className(String name) { - super(name); - } - - @Override - protected void setUp() { + @Before + public void setUp() { try { qt.clearTestSideEffects(); - } - catch (Throwable e) { + } catch (Throwable e) { e.printStackTrace(); System.err.flush(); fail("Unexpected exception in setup"); } } - @Override - protected void tearDown() { + @After + public void tearDown() { try { qt.clearPostTestEffects(); - if (getName().equals("testNegativeCliDriver_shutdown")) - qt.shutdown(); - } - catch (Exception e) { + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); @@ -90,24 +75,16 @@ public class $className extends TestCase { } } - public static Test suite() { - Set<String> qFilesToExecute = new HashSet<String>(); - String qFiles = System.getProperty("qfile", "").trim(); - if(!qFiles.isEmpty()) { - for(String qFile : qFiles.split(",")) { - qFile = qFile.trim(); - if(!qFile.isEmpty()) { - qFilesToExecute.add(qFile); - } - } + @AfterClass + public static void shutdown() throws Exception { + try { + qt.shutdown(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in shutdown"); } - - TestSuite suite = new TestSuite(); - - QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, - suite, null, new TestNegativeCliDriverAddTestFromQFiles()); - suite.addTest(new $className("testNegativeCliDriver_shutdown")); - return suite; } /** @@ -125,6 +102,7 @@ public class $className extends TestCase { #set ($eidx = $fname.indexOf('.')) #set ($tname = $fname.substring(0, $eidx)) #set ($fpath = $qfilesMap.get($fname)) + @Test public void testNegativeCliDriver_$tname() throws Exception { runTest("$tname", "$fname", (HIVE_ROOT + "$fpath")); } http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/ql/src/test/templates/TestParseNegative.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestParseNegative.vm b/ql/src/test/templates/TestParseNegative.vm index c5e7bdf..a4397f7 100755 --- a/ql/src/test/templates/TestParseNegative.vm +++ b/ql/src/test/templates/TestParseNegative.vm @@ -17,27 +17,23 @@ */ package org.apache.hadoop.hive.ql.parse; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - import java.io.*; import java.util.*; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.ql.exec.Task; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -public class $className extends TestCase { +public class $className { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; - - public static class TestParseNegativeAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { - public void addTestToSuite(TestSuite suite, Object setup, String tName) { - suite.addTest(new $className("testParseNegative_"+tName)); - } - } static { @@ -57,18 +53,11 @@ public class $className extends TestCase { } } - public $className(String name) { - super(name); - } - - @Override - protected void tearDown() { + @After + public void tearDown() { try { qt.clearPostTestEffects(); - if (getName().equals("testParseNegative_shutdown")) - qt.shutdown(); - } - catch (Exception e) { + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); @@ -76,31 +65,16 @@ public class $className extends TestCase { } } - /** - * Dummy last test. This is only meant to shutdown qt - */ - public void testParseNegative_shutdown() { - System.err.println ("Cleaning up " + "$className"); - } - - public static Test suite() { - Set<String> qFilesToExecute = new HashSet<String>(); - String qFiles = System.getProperty("qfile", "").trim(); - if(!qFiles.isEmpty()) { - for(String qFile : qFiles.split(",")) { - qFile = qFile.trim(); - if(!qFile.isEmpty()) { - qFilesToExecute.add(qFile); - } - } + @AfterClass + public static void shutdown() throws Exception { + try { + qt.shutdown(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in shutdown"); } - - TestSuite suite = new TestSuite(); - - QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, - suite, null, new TestParseNegativeAddTestFromQFiles()); - suite.addTest(new $className("testParseNegative_shutdown")); - return suite; } static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " @@ -111,6 +85,7 @@ public class $className extends TestCase { #set ($eidx = $fname.indexOf('.')) #set ($tname = $fname.substring(0, $eidx)) #set ($fpath = $qfilesMap.get($fname)) + @Test public void testParseNegative_$tname() throws Exception { runTest("$tname", "$fname", (HIVE_ROOT + "$fpath")); } http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java ---------------------------------------------------------------------- diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java b/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java index af6332e..c9c975b 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java @@ -55,9 +55,9 @@ public interface JobContext { Map<String, List<JavaFutureAction<?>>> getMonitoredJobs(); /** - * Return all added jar path which added through AddJarJob. + * Return all added jar path and timestamp which added through AddJarJob. */ - Set<String> getAddedJars(); + Map<String, Long> getAddedJars(); /** * Returns a local tmp dir specific to the context http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java ---------------------------------------------------------------------- diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java index beed8a3..b73bcd7 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java @@ -18,12 +18,10 @@ package org.apache.hive.spark.client; import java.io.File; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; import org.apache.hive.spark.counter.SparkCounters; @@ -35,14 +33,14 @@ class JobContextImpl implements JobContext { private final JavaSparkContext sc; private final ThreadLocal<MonitorCallback> monitorCb; private final Map<String, List<JavaFutureAction<?>>> monitoredJobs; - private final Set<String> addedJars; + private final Map<String, Long> addedJars; private final File localTmpDir; public JobContextImpl(JavaSparkContext sc, File localTmpDir) { this.sc = sc; this.monitorCb = new ThreadLocal<MonitorCallback>(); monitoredJobs = new ConcurrentHashMap<String, List<JavaFutureAction<?>>>(); - addedJars = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); + addedJars = new ConcurrentHashMap<>(); this.localTmpDir = localTmpDir; } @@ -65,7 +63,7 @@ class JobContextImpl implements JobContext { } @Override - public Set<String> getAddedJars() { + public Map<String, Long> getAddedJars() { return addedJars; } http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java ---------------------------------------------------------------------- diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java index b77c9e8..f5b1e48 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java @@ -43,26 +43,13 @@ import org.apache.hive.spark.client.metrics.Metrics; import org.apache.hive.spark.client.rpc.Rpc; import org.apache.hive.spark.client.rpc.RpcConfiguration; import org.apache.hive.spark.counter.SparkCounters; +import org.apache.spark.JavaSparkListener; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaFutureAction; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.scheduler.SparkListener; -import org.apache.spark.scheduler.SparkListenerApplicationEnd; -import org.apache.spark.scheduler.SparkListenerApplicationStart; -import org.apache.spark.scheduler.SparkListenerBlockManagerAdded; -import org.apache.spark.scheduler.SparkListenerBlockManagerRemoved; -import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate; -import org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate; import org.apache.spark.scheduler.SparkListenerJobEnd; import org.apache.spark.scheduler.SparkListenerJobStart; -import org.apache.spark.scheduler.SparkListenerStageCompleted; -import org.apache.spark.scheduler.SparkListenerStageSubmitted; import org.apache.spark.scheduler.SparkListenerTaskEnd; -import org.apache.spark.scheduler.SparkListenerTaskGettingResult; -import org.apache.spark.scheduler.SparkListenerTaskStart; -import org.apache.spark.scheduler.SparkListenerUnpersistRDD; -import org.apache.spark.scheduler.SparkListenerExecutorRemoved; -import org.apache.spark.scheduler.SparkListenerExecutorAdded; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -438,21 +425,11 @@ public class RemoteDriver { } - private class ClientListener implements SparkListener { + private class ClientListener extends JavaSparkListener { private final Map<Integer, Integer> stageToJobId = Maps.newHashMap(); @Override - public void onExecutorRemoved(SparkListenerExecutorRemoved removed) { - - } - - @Override - public void onExecutorAdded(SparkListenerExecutorAdded added) { - - } - - @Override public void onJobStart(SparkListenerJobStart jobStart) { synchronized (stageToJobId) { for (int i = 0; i < jobStart.stageIds().length(); i++) { @@ -500,39 +477,6 @@ public class RemoteDriver { } } - @Override - public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { } - - @Override - public void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) { } - - @Override - public void onTaskStart(SparkListenerTaskStart taskStart) { } - - @Override - public void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) { } - - @Override - public void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) { } - - @Override - public void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) { } - - @Override - public void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) { } - - @Override - public void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) { } - - @Override - public void onApplicationStart(SparkListenerApplicationStart applicationStart) { } - - @Override - public void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) { } - - @Override - public void onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate executorMetricsUpdate) { } - /** * Returns the client job ID for the given Spark job ID. * http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java ---------------------------------------------------------------------- diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java index ceebbb3..3d682a0 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java @@ -617,7 +617,7 @@ class SparkClientImpl implements SparkClient { jc.sc().addJar(path); // Following remote job may refer to classes in this jar, and the remote job would be executed // in a different thread, so we add this jar path to JobContext for further usage. - jc.getAddedJars().add(path); + jc.getAddedJars().put(path, System.currentTimeMillis()); return null; } http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java ---------------------------------------------------------------------- diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java index cd38346..b779f3f 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java @@ -24,7 +24,8 @@ import java.io.File; import java.net.URL; import java.net.URLClassLoader; import java.util.List; -import java.util.Set; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -36,19 +37,21 @@ import org.apache.hadoop.fs.Path; public class SparkClientUtilities { protected static final transient Logger LOG = LoggerFactory.getLogger(SparkClientUtilities.class); + private static final Map<String, Long> downloadedFiles = new ConcurrentHashMap<>(); + /** * Add new elements to the classpath. * - * @param newPaths Set of classpath elements + * @param newPaths Map of classpath elements and corresponding timestamp */ - public static void addToClassPath(Set<String> newPaths, Configuration conf, File localTmpDir) + public static void addToClassPath(Map<String, Long> newPaths, Configuration conf, File localTmpDir) throws Exception { URLClassLoader loader = (URLClassLoader) Thread.currentThread().getContextClassLoader(); List<URL> curPath = Lists.newArrayList(loader.getURLs()); boolean newPathAdded = false; - for (String newPath : newPaths) { - URL newUrl = urlFromPathString(newPath, conf, localTmpDir); + for (Map.Entry<String, Long> entry : newPaths.entrySet()) { + URL newUrl = urlFromPathString(entry.getKey(), entry.getValue(), conf, localTmpDir); if (newUrl != null && !curPath.contains(newUrl)) { curPath.add(newUrl); LOG.info("Added jar[" + newUrl + "] to classpath."); @@ -69,7 +72,8 @@ public class SparkClientUtilities { * @param path path string * @return */ - private static URL urlFromPathString(String path, Configuration conf, File localTmpDir) { + private static URL urlFromPathString(String path, Long timeStamp, + Configuration conf, File localTmpDir) { URL url = null; try { if (StringUtils.indexOf(path, "file:/") == 0) { @@ -78,12 +82,17 @@ public class SparkClientUtilities { Path remoteFile = new Path(path); Path localFile = new Path(localTmpDir.getAbsolutePath() + File.separator + remoteFile.getName()); - if (!new File(localFile.toString()).exists()) { + Long currentTS = downloadedFiles.get(path); + if (currentTS == null) { + currentTS = -1L; + } + if (!new File(localFile.toString()).exists() || currentTS < timeStamp) { LOG.info("Copying " + remoteFile + " to " + localFile); FileSystem remoteFS = remoteFile.getFileSystem(conf); remoteFS.copyToLocalFile(remoteFile, localFile); + downloadedFiles.put(path, timeStamp); } - return urlFromPathString(localFile.toString(), conf, localTmpDir); + return urlFromPathString(localFile.toString(), timeStamp, conf, localTmpDir); } else { url = new File(path).toURL(); } http://git-wip-us.apache.org/repos/asf/hive/blob/0a88760f/testutils/ptest2/src/main/resources/batch-exec.vm ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/resources/batch-exec.vm b/testutils/ptest2/src/main/resources/batch-exec.vm index c155851..da3e0ac 100644 --- a/testutils/ptest2/src/main/resources/batch-exec.vm +++ b/testutils/ptest2/src/main/resources/batch-exec.vm @@ -62,8 +62,6 @@ then testModule=./ fi pushd $testModule - #clean to force regeneration of class files (maven sometimes skips generation) - mvn clean -Dmaven.repo.local=$localDir/$instanceName/maven $mavenArgs timeout 2h mvn -B test -Dmaven.repo.local=$localDir/$instanceName/maven \ $mavenArgs $mavenTestArgs $testArguments 1>$logDir/maven-test.txt 2>&1 </dev/null & #[[
