http://git-wip-us.apache.org/repos/asf/hive/blob/677e5d20/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out index a628aae..16aa452 100644 --- a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out @@ -62,10 +62,10 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) - outputColumnNames: _col0 + outputColumnNames: ds Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string) + keys: ds (type: string) mode: hash outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE @@ -213,16 +213,20 @@ STAGE PLANS: expressions: ds (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: ds + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - target column name: ds - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: ds + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + target column name: ds + target work: Map 1 Stage: Stage-1 Spark @@ -237,11 +241,15 @@ STAGE PLANS: alias: srcpart filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -251,19 +259,23 @@ STAGE PLANS: Filter Operator predicate: (ds is not null and (date = '2008-04-08')) (type: boolean) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string) - 1 ds (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -336,11 +348,15 @@ STAGE PLANS: alias: srcpart filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -350,19 +366,23 @@ STAGE PLANS: Filter Operator predicate: (ds is not null and (date = '2008-04-08')) (type: boolean) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string) - 1 ds (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -457,39 +477,47 @@ STAGE PLANS: expressions: ds (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: ds + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - target column name: ds - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: ds + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + target column name: ds + target work: Map 1 Map 8 Map Operator Tree: TableScan alias: srcpart_hour - filterExpr: (hr is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: hr + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - target column name: hr - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: hr + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + target column name: hr + target work: Map 1 Stage: Stage-1 Spark @@ -504,12 +532,16 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: hr (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Map 5 Map Operator Tree: TableScan @@ -519,39 +551,47 @@ STAGE PLANS: Filter Operator predicate: (ds is not null and (date = '2008-04-08')) (type: boolean) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan alias: srcpart_hour - filterExpr: (hr is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: hr (type: string) - sort order: + - Map-reduce partition columns: hr (type: string) + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string) - 1 ds (type: string) - outputColumnNames: _col3 + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col3 (type: string) + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -559,8 +599,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col3 (type: string) - 1 hr (type: string) + 0 _col1 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -640,12 +680,16 @@ STAGE PLANS: alias: srcpart filterExpr: (ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: hr (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Map 5 Map Operator Tree: TableScan @@ -655,39 +699,47 @@ STAGE PLANS: Filter Operator predicate: (ds is not null and (date = '2008-04-08')) (type: boolean) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan alias: srcpart_hour - filterExpr: (hr is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: hr (type: string) - sort order: + - Map-reduce partition columns: hr (type: string) + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string) - 1 ds (type: string) - outputColumnNames: _col3 + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col3 (type: string) + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -695,8 +747,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col3 (type: string) - 1 hr (type: string) + 0 _col1 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -782,48 +834,56 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_date_hour - filterExpr: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + filterExpr: (ds is not null and hr is not null and (date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + predicate: (ds is not null and hr is not null and (date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: ds + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - target column name: ds - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: ds + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + target column name: ds + target work: Map 1 Map 6 Map Operator Tree: TableScan alias: srcpart_date_hour - filterExpr: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + filterExpr: (ds is not null and hr is not null and (date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + predicate: (ds is not null and hr is not null and (date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col2 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: hr + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - target column name: hr - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: hr + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + target column name: hr + target work: Map 1 Stage: Stage-1 Spark @@ -837,33 +897,41 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string), hr (type: string) - sort order: ++ - Map-reduce partition columns: ds (type: string), hr (type: string) + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart_date_hour - filterExpr: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + filterExpr: (ds is not null and hr is not null and (date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + predicate: (ds is not null and hr is not null and (date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string), hr (type: string) - sort order: ++ - Map-reduce partition columns: ds (type: string), hr (type: string) + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string), hr (type: string) - 1 ds (type: string), hr (type: string) + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -936,33 +1004,41 @@ STAGE PLANS: alias: srcpart filterExpr: (ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string), hr (type: string) - sort order: ++ - Map-reduce partition columns: ds (type: string), hr (type: string) + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart_date_hour - filterExpr: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + filterExpr: (ds is not null and hr is not null and (date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + predicate: (ds is not null and hr is not null and (date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string), hr (type: string) - sort order: ++ - Map-reduce partition columns: ds (type: string), hr (type: string) + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string), hr (type: string) - 1 ds (type: string), hr (type: string) + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1053,16 +1129,20 @@ STAGE PLANS: expressions: ds (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: ds + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - target column name: ds - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: ds + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + target column name: ds + target work: Map 1 Stage: Stage-1 Spark @@ -1077,11 +1157,15 @@ STAGE PLANS: alias: srcpart filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -1091,19 +1175,23 @@ STAGE PLANS: Filter Operator predicate: (ds is not null and (date = 'I DONT EXIST')) (type: boolean) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string) - 1 ds (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1176,11 +1264,15 @@ STAGE PLANS: alias: srcpart filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -1190,19 +1282,23 @@ STAGE PLANS: Filter Operator predicate: (ds is not null and (date = 'I DONT EXIST')) (type: boolean) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string) - 1 ds (type: string) + 0 _col0 (type: string) + 1 _col0 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1282,25 +1378,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_double_hour - filterExpr: (UDFToDouble(UDFToInteger((hr / 2))) is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToDouble(UDFToInteger((hr / 2))) is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((hr / 2))) (type: double) + expressions: hr (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash + Select Operator + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: UDFToDouble(hr) + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - target column name: hr - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: UDFToDouble(hr) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + target column name: hr + target work: Map 1 Stage: Stage-1 Spark @@ -1313,39 +1413,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: UDFToDouble(hr) is not null (type: boolean) + filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: UDFToDouble(hr) is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(hr) (type: double) + key expressions: UDFToDouble(_col0) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(hr) (type: double) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart_double_hour - filterExpr: (UDFToDouble(UDFToInteger((hr / 2))) is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToDouble(UDFToInteger((hr / 2))) is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((hr / 2))) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((hr / 2))) (type: double) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 UDFToDouble(hr) (type: double) - 1 UDFToDouble(UDFToInteger((hr / 2))) (type: double) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1413,25 +1518,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_double_hour - filterExpr: (hr is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash + Select Operator + expressions: _col0 (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: (hr * 2) + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - target column name: hr - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: (UDFToDouble(hr) * 2.0) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + target column name: hr + target work: Map 1 Stage: Stage-1 Spark @@ -1444,39 +1553,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (hr * 2) is not null (type: boolean) + filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (hr * 2) is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (hr * 2) (type: double) + key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) sort order: + - Map-reduce partition columns: (hr * 2) (type: double) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart_double_hour - filterExpr: (hr is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: hr (type: double) - sort order: + - Map-reduce partition columns: hr (type: double) + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 (hr * 2) (type: double) - 1 hr (type: double) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 1 _col0 (type: double) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1546,39 +1660,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: UDFToDouble(hr) is not null (type: boolean) + filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: UDFToDouble(hr) is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(hr) (type: double) + key expressions: UDFToDouble(_col0) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(hr) (type: double) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart_double_hour - filterExpr: (UDFToDouble(UDFToInteger((hr / 2))) is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToDouble(UDFToInteger((hr / 2))) is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((hr / 2))) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((hr / 2))) (type: double) + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 UDFToDouble(hr) (type: double) - 1 UDFToDouble(UDFToInteger((hr / 2))) (type: double) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1648,39 +1767,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (hr * 2) is not null (type: boolean) + filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (hr * 2) is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (hr * 2) (type: double) + key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) sort order: + - Map-reduce partition columns: (hr * 2) (type: double) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart_double_hour - filterExpr: (hr is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: hr (type: double) - sort order: + - Map-reduce partition columns: hr (type: double) + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 (hr * 2) (type: double) - 1 hr (type: double) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 1 _col0 (type: double) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1761,25 +1885,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_double_hour - filterExpr: (UDFToString(hr) is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToString(hr) is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToString(hr) (type: string) + expressions: hr (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: UDFToString(_col0) (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: UDFToString((hr * 2)) + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - target column name: hr - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + target column name: hr + target work: Map 1 Stage: Stage-1 Spark @@ -1792,39 +1920,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: UDFToString((hr * 2)) is not null (type: boolean) + filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: UDFToString((hr * 2)) is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToString((hr * 2)) (type: string) + key expressions: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) sort order: + - Map-reduce partition columns: UDFToString((hr * 2)) (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart_double_hour - filterExpr: (UDFToString(hr) is not null and (hour = 11)) (type: boolean) + filterExpr: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToString(hr) is not null and (hour = 11)) (type: boolean) + predicate: (hr is not null and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToString(hr) (type: string) - sort order: + - Map-reduce partition columns: UDFToString(hr) (type: string) + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToString(_col0) (type: string) + sort order: + + Map-reduce partition columns: UDFToString(_col0) (type: string) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 UDFToString((hr * 2)) (type: string) - 1 UDFToString(hr) (type: string) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + 1 UDFToString(_col0) (type: string) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1887,7 +2020,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as date from srcpart group by ds) s on (srcpart.ds = s.ds) where s.date = '2008-04-08' PREHOOK: type: QUERY @@ -1895,15 +2027,64 @@ POSTHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as date from srcpart group by ds) s on (srcpart.ds = s.ds) where s.date = '2008-04-08' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-2 + Spark + Edges: + Reducer 7 <- Map 6 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 6 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '2008-04-08' (type: string) + outputColumnNames: ds + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: ds (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reducer 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Spark Partition Pruning Sink Operator + partition key expr: ds + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + target column name: ds + target work: Map 1 + Stage: Stage-1 Spark Edges: Reducer 5 <- Map 4 (GROUP, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1911,13 +2092,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -1926,10 +2111,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 + outputColumnNames: ds Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string) + keys: ds (type: string) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE @@ -1944,9 +2129,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 - 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1977,11 +2162,11 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -1989,18 +2174,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as date from srcpart group by ds) s on (srcpart.ds = s.ds) where s.date = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as date from srcpart group by ds) s on (srcpart.ds = s.ds) where s.date = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2016,7 +2204,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 -Warning: Shuffle Join JOIN[4][tables = [srcpart, srcpart_date_hour]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: -- non-equi join EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.date = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY @@ -2040,23 +2228,31 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: ds (type: string), hr (type: string) + Reduce Output Operator + sort order: + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: srcpart_date_hour - filterExpr: ((date = '2008-04-08') and (hour = 11)) (type: boolean) + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((date = '2008-04-08') and (hour = 11)) (type: boolean) + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - value expressions: ds (type: string), hr (type: string) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col2 (type: string) Reducer 2 Reduce Operator Tree: Join Operator @@ -2065,10 +2261,10 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col2, _col3, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col4 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col2 = _col7) or (_col3 = _col9)) (type: boolean) + predicate: ((_col0 = _col2) or (_col1 = _col4)) (type: boolean) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -2102,7 +2298,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[4][tables = [srcpart, srcpart_date_hour]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.date = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2142,48 +2338,56 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_date_hour - filterExpr: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: ds + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - target column name: ds - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: ds + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + target column name: ds + target work: Map 1 Map 6 Map Operator Tree: TableScan alias: srcpart_date_hour - filterExpr: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col2 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Spark Partition Pruning Sink Operator - partition key expr: hr + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - target column name: hr - target work: Map 1 + Spark Partition Pruning Sink Operator + partition key expr: hr + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + target column name: hr + target work: Map 1 Stage: Stage-1 Spark @@ -2197,49 +2401,51 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string), hr (type: string) - sort order: ++ - Map-reduce partition columns: ds (type: string), hr (type: string) + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart_date_hour - filterExpr: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((ds is not null and hr is not null) and (date = '2008-04-08')) and (hour = 11)) (type: boolean) + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string), hr (type: string) - sort order: ++ - Map-reduce partition columns: ds (type: string), hr (type: string) + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string), hr (type: string) - 1 ds (type: string), hr (type: string) - outputColumnNames: _col2, _col3, _col7, _col9 + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((_col2 = _col7) and (_col3 = _col9)) (type: boolean) - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Reducer 3 Reduce Operator Tree: Group By Operator @@ -2287,62 +2493,101 @@ POSTHOOK: query: -- left join EXPLAIN select count(*) from srcpart left join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpa
<TRUNCATED>
