Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/vector_decimal_aggregate.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/vector_decimal_aggregate.q?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/vector_decimal_aggregate.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/vector_decimal_aggregate.q Wed Sep 17 05:28:43 2014 @@ -1,20 +1,35 @@ CREATE TABLE decimal_vgby STORED AS ORC AS - SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, - CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, - cint - FROM alltypesorc; + SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, + CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, + cint + FROM alltypesorc; SET hive.vectorized.execution.enabled=true; +-- First only do simple aggregations that output primitives only EXPLAIN SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1; + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1; SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1; \ No newline at end of file + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1; + +-- Now add the others... +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1; +SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1; \ No newline at end of file
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_decimal_aggregate.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_decimal_aggregate.q.out?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_decimal_aggregate.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_decimal_aggregate.q.out Wed Sep 17 05:28:43 2014 @@ -1,34 +1,141 @@ PREHOOK: query: CREATE TABLE decimal_vgby STORED AS ORC AS - SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, - CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, - cint - FROM alltypesorc + SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, + CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, + cint + FROM alltypesorc PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@decimal_vgby POSTHOOK: query: CREATE TABLE decimal_vgby STORED AS ORC AS - SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, - CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, - cint - FROM alltypesorc + SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, + CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, + cint + FROM alltypesorc POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@decimal_vgby -PREHOOK: query: EXPLAIN SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1 +PREHOOK: query: -- First only do simple aggregations that output primitives only +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1 +POSTHOOK: query: -- First only do simple aggregations that output primitives only +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: decimal_vgby + Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)) + outputColumnNames: cint, cdecimal1, cdecimal2 + Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cdecimal1), max(cdecimal1), min(cdecimal1), sum(cdecimal1), count(cdecimal2), max(cdecimal2), min(cdecimal2), sum(cdecimal2), count() + keys: cint (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)), _col9 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), sum(VALUE._col3), count(VALUE._col4), max(VALUE._col5), min(VALUE._col6), sum(VALUE._col7), count(VALUE._col8) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col9 > 1) (type: boolean) + Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_vgby +#### A masked pattern was here #### +POSTHOOK: query: SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_vgby +#### A masked pattern was here #### +0 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.71538461538500 -5147.90769230769300 6010604.30769230735360 +-3728 6 5831542.2692483780 -3367.6517567568 5817556.0411483778 6 6984454.21109769200000 -4033.445769230769 6967702.86724384584710 +-563 2 -515.6210729730 -3367.6517567568 -3883.2728297298 2 -617.56077692307690 -4033.445769230769 -4651.00654615384590 +762 2 5831542.2692483780 1531.2194054054 5833073.4886537834 2 6984454.21109769200000 1833.9456923076925 6986288.15678999969250 +6981 3 5831542.269248378 -515.6210729730 5830511.0271024320 3 6984454.211097692 -617.56077692307690 6983219.08954384584620 +253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.96923076923100 -11712.99230769231000 -416182.64030769233089 +528534767 1024 5831542.2692483780 -9777.1594594595 11646372.8607481068 1024 6984454.21109769200000 -11710.13076923077100 13948892.79980307629003 +626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.74615384615400 -11712.27692307692300 12625.04759999997746 +PREHOOK: query: -- Now add the others... +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 +PREHOOK: type: QUERY +POSTHOOK: query: -- Now add the others... +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -93,20 +200,20 @@ STAGE PLANS: ListSink PREHOOK: query: SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1 + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 PREHOOK: type: QUERY PREHOOK: Input: default@decimal_vgby #### A masked pattern was here #### POSTHOOK: query: SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1 + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### Modified: hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out Wed Sep 17 05:28:43 2014 @@ -1,34 +1,134 @@ PREHOOK: query: CREATE TABLE decimal_vgby STORED AS ORC AS - SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, - CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, - cint - FROM alltypesorc + SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, + CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, + cint + FROM alltypesorc PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@decimal_vgby POSTHOOK: query: CREATE TABLE decimal_vgby STORED AS ORC AS - SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, - CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, - cint - FROM alltypesorc + SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, + CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, + cint + FROM alltypesorc POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@decimal_vgby -PREHOOK: query: EXPLAIN SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1 +PREHOOK: query: -- First only do simple aggregations that output primitives only +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1 +POSTHOOK: query: -- First only do simple aggregations that output primitives only +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: decimal_vgby + Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)) + outputColumnNames: cint, cdecimal1, cdecimal2 + Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cdecimal1), max(cdecimal1), min(cdecimal1), sum(cdecimal1), count(cdecimal2), max(cdecimal2), min(cdecimal2), sum(cdecimal2), count() + keys: cint (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)), _col9 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), sum(VALUE._col3), count(VALUE._col4), max(VALUE._col5), min(VALUE._col6), sum(VALUE._col7), count(VALUE._col8) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col9 > 1) (type: boolean) + Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_vgby +#### A masked pattern was here #### +POSTHOOK: query: SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_vgby +#### A masked pattern was here #### +NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.71538461538500 -5147.90769230769300 6010604.30769230735360 +-3728 6 5831542.2692483780 -3367.6517567568 5817556.0411483778 6 6984454.21109769200000 -4033.445769230769 6967702.86724384584710 +-563 2 -515.6210729730 -3367.6517567568 -3883.2728297298 2 -617.56077692307690 -4033.445769230769 -4651.00654615384590 +762 2 5831542.2692483780 1531.2194054054 5833073.4886537834 2 6984454.21109769200000 1833.9456923076925 6986288.15678999969250 +6981 3 5831542.269248378 -515.6210729730 5830511.0271024320 3 6984454.211097692 -617.56077692307690 6983219.08954384584620 +253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.96923076923100 -11712.99230769231000 -416182.64030769233089 +528534767 1024 5831542.2692483780 -9777.1594594595 11646372.8607481068 1024 6984454.21109769200000 -11710.13076923077100 13948892.79980307629003 +626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.74615384615400 -11712.27692307692300 12625.04759999997746 +PREHOOK: query: -- Now add the others... +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 +PREHOOK: type: QUERY +POSTHOOK: query: -- Now add the others... +EXPLAIN SELECT cint, + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -87,20 +187,20 @@ STAGE PLANS: ListSink PREHOOK: query: SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1 + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 PREHOOK: type: QUERY PREHOOK: Input: default@decimal_vgby #### A masked pattern was here #### POSTHOOK: query: SELECT cint, - COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), - COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) - FROM decimal_vgby - GROUP BY cint - HAVING COUNT(*) > 1 + COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), + COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2), AVG(cdecimal2), STDDEV_POP(cdecimal2), STDDEV_SAMP(cdecimal2) + FROM decimal_vgby + GROUP BY cint + HAVING COUNT(*) > 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### Modified: hive/branches/cbo/serde/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/pom.xml?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/serde/pom.xml (original) +++ hive/branches/cbo/serde/pom.xml Wed Sep 17 05:28:43 2014 @@ -70,7 +70,13 @@ <artifactId>libthrift</artifactId> <version>${libthrift.version}</version> </dependency> - <!-- test inter-project --> + <dependency> + <groupId>net.sf.opencsv</groupId> + <artifactId>opencsv</artifactId> + <version>${opencsv.version}</version> + </dependency> + + <!-- test inter-project --> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java (original) +++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java Wed Sep 17 05:28:43 2014 @@ -66,7 +66,7 @@ public class CLIService extends Composit private UserGroupInformation httpUGI; public CLIService() { - super("CLIService"); + super(CLIService.class.getSimpleName()); } @Override @@ -201,8 +201,7 @@ public class CLIService extends Composit * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle) */ @Override - public void closeSession(SessionHandle sessionHandle) - throws HiveSQLException { + public void closeSession(SessionHandle sessionHandle) throws HiveSQLException { sessionManager.closeSession(sessionHandle); LOG.debug(sessionHandle + ": closeSession()"); } @@ -470,4 +469,8 @@ public class CLIService extends Composit sessionManager.getSession(sessionHandle).renewDelegationToken(authFactory, tokenStr); LOG.info(sessionHandle + ": renewDelegationToken()"); } + + public SessionManager getSessionManager() { + return sessionManager; + } } Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java (original) +++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java Wed Sep 17 05:28:43 2014 @@ -47,7 +47,7 @@ public class OperationManager extends Ab new HashMap<OperationHandle, Operation>(); public OperationManager() { - super("OperationManager"); + super(OperationManager.class.getSimpleName()); } @Override Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original) +++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Wed Sep 17 05:28:43 2014 @@ -166,8 +166,8 @@ public class HiveSessionImpl implements IHiveFileProcessor processor = new GlobalHivercFileProcessor(); try { - if (hiveConf.getVar(ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION) != null) { - String hiverc = hiveConf.getVar(ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION) + if (hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION) != null) { + String hiverc = hiveConf.getVar(ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION) + File.separator + SessionManager.HIVERCFILE; if (new File(hiverc).exists()) { LOG.info("Running global init file: " + hiverc); Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java (original) +++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java Wed Sep 17 05:28:43 2014 @@ -67,7 +67,7 @@ public class SessionManager extends Comp private volatile boolean shutdown; public SessionManager() { - super("SessionManager"); + super(SessionManager.class.getSimpleName()); } @Override @@ -356,5 +356,9 @@ public class SessionManager extends Comp return backgroundOperationPool.submit(r); } + public int getOpenSessionCount() { + return handleToSession.size(); + } + } Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java (original) +++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java Wed Sep 17 05:28:43 2014 @@ -18,7 +18,6 @@ package org.apache.hive.service.cli.thrift; -import java.net.InetSocketAddress; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -40,72 +39,54 @@ import org.apache.thrift.transport.TTran public class ThriftBinaryCLIService extends ThriftCLIService { public ThriftBinaryCLIService(CLIService cliService) { - super(cliService, "ThriftBinaryCLIService"); + super(cliService, ThriftBinaryCLIService.class.getSimpleName()); } @Override public void run() { try { - hiveAuthFactory = new HiveAuthFactory(hiveConf); - TTransportFactory transportFactory = hiveAuthFactory.getAuthTransFactory(); - TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this); - - String portString = System.getenv("HIVE_SERVER2_THRIFT_PORT"); - if (portString != null) { - portNum = Integer.valueOf(portString); - } else { - portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT); - } - - String hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST"); - if (hiveHost == null) { - hiveHost = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST); - } - - if (hiveHost != null && !hiveHost.isEmpty()) { - serverAddress = new InetSocketAddress(hiveHost, portNum); - } else { - serverAddress = new InetSocketAddress(portNum); - } - - minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS); - maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS); - workerKeepAliveTime = hiveConf.getTimeVar( - ConfVars.HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS); + // Server thread pool String threadPoolName = "HiveServer2-Handler-Pool"; ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads, workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new ThreadFactoryWithGarbageCleanup(threadPoolName)); + // Thrift configs + hiveAuthFactory = new HiveAuthFactory(hiveConf); + TTransportFactory transportFactory = hiveAuthFactory.getAuthTransFactory(); + TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this); TServerSocket serverSocket = null; if (!hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL)) { serverSocket = HiveAuthFactory.getServerSocket(hiveHost, portNum); } else { String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim(); if (keyStorePath.isEmpty()) { - throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + - " Not configured for SSL connection"); + throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + + " Not configured for SSL connection"); } String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf, HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname); - serverSocket = HiveAuthFactory.getServerSSLSocket(hiveHost, portNum, - keyStorePath, keyStorePassword); + serverSocket = HiveAuthFactory.getServerSSLSocket(hiveHost, portNum, keyStorePath, + keyStorePassword); } + + // Server args TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket) - .processorFactory(processorFactory) - .transportFactory(transportFactory) - .protocolFactory(new TBinaryProtocol.Factory()) - .executorService(executorService); + .processorFactory(processorFactory).transportFactory(transportFactory) + .protocolFactory(new TBinaryProtocol.Factory()).executorService(executorService); + // TCP Server server = new TThreadPoolServer(sargs); - - LOG.info("ThriftBinaryCLIService listening on " + serverAddress); - server.serve(); - + String msg = "Started " + ThriftBinaryCLIService.class.getSimpleName() + " on port " + + portNum + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads"; + LOG.info(msg); } catch (Throwable t) { - LOG.error("Error: ", t); + LOG.fatal( + "Error starting HiveServer2: could not start " + + ThriftBinaryCLIService.class.getSimpleName(), t); + System.exit(-1); } - } + } Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java (original) +++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java Wed Sep 17 05:28:43 2014 @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import javax.security.auth.login.LoginException; @@ -34,6 +35,7 @@ import org.apache.hive.service.auth.Hive import org.apache.hive.service.auth.TSetIpAddressProcessor; import org.apache.hive.service.cli.*; import org.apache.hive.service.cli.session.SessionManager; +import org.apache.hive.service.server.HiveServer2; import org.apache.thrift.TException; import org.apache.thrift.server.TServer; @@ -48,9 +50,11 @@ public abstract class ThriftCLIService e protected CLIService cliService; private static final TStatus OK_STATUS = new TStatus(TStatusCode.SUCCESS_STATUS); private static final TStatus ERROR_STATUS = new TStatus(TStatusCode.ERROR_STATUS); + protected static HiveAuthFactory hiveAuthFactory; protected int portNum; protected InetSocketAddress serverAddress; + protected String hiveHost; protected TServer server; protected org.eclipse.jetty.server.Server httpServer; @@ -62,8 +66,7 @@ public abstract class ThriftCLIService e protected int minWorkerThreads; protected int maxWorkerThreads; protected long workerKeepAliveTime; - - protected static HiveAuthFactory hiveAuthFactory; + private HiveServer2 hiveServer2; public ThriftCLIService(CLIService cliService, String serviceName) { super(serviceName); @@ -73,6 +76,43 @@ public abstract class ThriftCLIService e @Override public synchronized void init(HiveConf hiveConf) { this.hiveConf = hiveConf; + + // Initialize common server configs needed in both binary & http modes + String portString; + hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST"); + if (hiveHost == null) { + hiveHost = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST); + } + // HTTP mode + if (HiveServer2.isHTTPTransportMode(hiveConf)) { + workerKeepAliveTime = + hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME, + TimeUnit.SECONDS); + portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT"); + if (portString != null) { + portNum = Integer.valueOf(portString); + } else { + portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT); + } + } + // Binary mode + else { + workerKeepAliveTime = + hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS); + portString = System.getenv("HIVE_SERVER2_THRIFT_PORT"); + if (portString != null) { + portNum = Integer.valueOf(portString); + } else { + portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT); + } + } + if (hiveHost != null && !hiveHost.isEmpty()) { + serverAddress = new InetSocketAddress(hiveHost, portNum); + } else { + serverAddress = new InetSocketAddress(portNum); + } + minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS); + maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS); super.init(hiveConf); } @@ -105,6 +145,14 @@ public abstract class ThriftCLIService e super.stop(); } + public int getPortNumber() { + return portNum; + } + + public InetSocketAddress getServerAddress() { + return serverAddress; + } + @Override public TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req) throws TException { @@ -308,6 +356,24 @@ public abstract class ThriftCLIService e } catch (Exception e) { LOG.warn("Error closing session: ", e); resp.setStatus(HiveSQLException.toTStatus(e)); + } finally { + if (!(isEmbedded) && (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) + && (!hiveServer2.isRegisteredWithZooKeeper())) { + // Asynchronously shutdown this instance of HiveServer2, + // if there are no active client sessions + if (cliService.getSessionManager().getOpenSessionCount() == 0) { + LOG.info("This instance of HiveServer2 has been removed from the list of server " + + "instances available for dynamic service discovery. " + + "The last client session has ended - will shutdown now."); + Thread shutdownThread = new Thread() { + @Override + public void run() { + hiveServer2.stop(); + } + }; + shutdownThread.start(); + } + } } return resp; } @@ -591,5 +657,9 @@ public abstract class ThriftCLIService e .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString()); } + public void setHiveServer2(HiveServer2 hiveServer2) { + this.hiveServer2 = hiveServer2; + } + } Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java (original) +++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java Wed Sep 17 05:28:43 2014 @@ -48,100 +48,94 @@ import org.eclipse.jetty.util.thread.Exe public class ThriftHttpCLIService extends ThriftCLIService { public ThriftHttpCLIService(CLIService cliService) { - super(cliService, "ThriftHttpCLIService"); + super(cliService, ThriftHttpCLIService.class.getSimpleName()); } + /** + * Configure Jetty to serve http requests. Example of a client connection URL: + * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ, + * e.g. http://gateway:port/hive2/servlets/thrifths2/ + */ @Override public void run() { try { - // Configure Jetty to serve http requests - // Example of a client connection URL: http://localhost:10000/servlets/thrifths2/ - // a gateway may cause actual target URL to differ, e.g. http://gateway:port/hive2/servlets/thrifths2/ - + // Verify config validity verifyHttpConfiguration(hiveConf); - String portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT"); - if (portString != null) { - portNum = Integer.valueOf(portString); - } else { - portNum = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT); - } - - minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MIN_WORKER_THREADS); - maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_WORKER_THREADS); - workerKeepAliveTime = hiveConf.getTimeVar( - ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS); - - String httpPath = getHttpPath(hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); - + // HTTP Server httpServer = new org.eclipse.jetty.server.Server(); + + // Server thread pool String threadPoolName = "HiveServer2-HttpHandler-Pool"; ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads, workerKeepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactoryWithGarbageCleanup(threadPoolName)); - ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService); httpServer.setThreadPool(threadPool); - SelectChannelConnector connector = new SelectChannelConnector();; + // Connector configs + SelectChannelConnector connector = new SelectChannelConnector(); boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL); String schemeName = useSsl ? "https" : "http"; - String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION); - // Set during the init phase of HiveServer2 if auth mode is kerberos - // UGI for the hive/_HOST (kerberos) principal - UserGroupInformation serviceUGI = cliService.getServiceUGI(); - // UGI for the http/_HOST (SPNego) principal - UserGroupInformation httpUGI = cliService.getHttpUGI(); - + // Change connector if SSL is used if (useSsl) { String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim(); String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf, HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname); if (keyStorePath.isEmpty()) { - throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + - " Not configured for SSL connection"); + throw new IllegalArgumentException(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname + + " Not configured for SSL connection"); } SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setKeyStorePath(keyStorePath); sslContextFactory.setKeyStorePassword(keyStorePassword); connector = new SslSelectChannelConnector(sslContextFactory); } - connector.setPort(portNum); // Linux:yes, Windows:no connector.setReuseAddress(!Shell.WINDOWS); - - int maxIdleTime = (int) hiveConf.getTimeVar( - ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME, TimeUnit.MILLISECONDS); + int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME, + TimeUnit.MILLISECONDS); connector.setMaxIdleTime(maxIdleTime); - + httpServer.addConnector(connector); + // Thrift configs hiveAuthFactory = new HiveAuthFactory(hiveConf); TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this); TProcessor processor = processorFactory.getProcessor(null); - TProtocolFactory protocolFactory = new TBinaryProtocol.Factory(); + // Set during the init phase of HiveServer2 if auth mode is kerberos + // UGI for the hive/_HOST (kerberos) principal + UserGroupInformation serviceUGI = cliService.getServiceUGI(); + // UGI for the http/_HOST (SPNego) principal + UserGroupInformation httpUGI = cliService.getHttpUGI(); + String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION); + TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, authType, + serviceUGI, httpUGI); - TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, - authType, serviceUGI, httpUGI); - + // Context handler final ServletContextHandler context = new ServletContextHandler( ServletContextHandler.SESSIONS); context.setContextPath("/"); - + String httpPath = getHttpPath(hiveConf + .getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); httpServer.setHandler(context); context.addServlet(new ServletHolder(thriftHttpServlet), httpPath); // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyRecieveDuration, etc. + // Finally, start the server httpServer.start(); - String msg = "Started ThriftHttpCLIService in " + schemeName + " mode on port " + portNum + - " path=" + httpPath + - " with " + minWorkerThreads + ".." + maxWorkerThreads + " worker threads"; + String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName + + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..." + + maxWorkerThreads + " worker threads"; LOG.info(msg); httpServer.join(); } catch (Throwable t) { - LOG.error("Error: ", t); + LOG.fatal( + "Error starting HiveServer2: could not start " + + ThriftHttpCLIService.class.getSimpleName(), t); + System.exit(-1); } } @@ -191,7 +185,8 @@ public class ThriftHttpCLIService extend // NONE in case of thrift mode uses SASL LOG.warn(ConfVars.HIVE_SERVER2_AUTHENTICATION + " setting to " + authType + ". SASL is not supported with http transport mode," + - " so using equivalent of " + AuthTypes.NOSASL); + " so using equivalent of " + + AuthTypes.NOSASL); } } Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/server/HiveServer2.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/server/HiveServer2.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/java/org/apache/hive/service/server/HiveServer2.java (original) +++ hive/branches/cbo/service/src/java/org/apache/hive/service/server/HiveServer2.java Wed Sep 17 05:28:43 2014 @@ -18,6 +18,8 @@ package org.apache.hive.service.server; +import java.nio.charset.Charset; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.LogUtils; @@ -25,12 +27,21 @@ import org.apache.hadoop.hive.common.Log import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; +import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.hive.common.util.HiveStringUtils; +import org.apache.hive.common.util.HiveVersionInfo; import org.apache.hive.service.CompositeService; +import org.apache.hive.service.ServiceException; import org.apache.hive.service.cli.CLIService; import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIService; import org.apache.hive.service.cli.thrift.ThriftHttpCLIService; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; /** * HiveServer2. @@ -41,9 +52,12 @@ public class HiveServer2 extends Composi private CLIService cliService; private ThriftCLIService thriftCLIService; + private String znodePath; + private ZooKeeper zooKeeperClient; + private boolean registeredWithZooKeeper = false; public HiveServer2() { - super("HiveServer2"); + super(HiveServer2.class.getSimpleName()); HiveConf.setLoadHiveServer2Config(true); } @@ -52,20 +66,129 @@ public class HiveServer2 extends Composi public synchronized void init(HiveConf hiveConf) { cliService = new CLIService(); addService(cliService); + if (isHTTPTransportMode(hiveConf)) { + thriftCLIService = new ThriftHttpCLIService(cliService); + } else { + thriftCLIService = new ThriftBinaryCLIService(cliService); + } + addService(thriftCLIService); + thriftCLIService.setHiveServer2(this); + super.init(hiveConf); + // Add a shutdown hook for catching SIGTERM & SIGINT + final HiveServer2 hiveServer2 = this; + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + hiveServer2.stop(); + } + }); + } + + public static boolean isHTTPTransportMode(HiveConf hiveConf) { String transportMode = System.getenv("HIVE_SERVER2_TRANSPORT_MODE"); - if(transportMode == null) { + if (transportMode == null) { transportMode = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE); } - if(transportMode != null && (transportMode.equalsIgnoreCase("http"))) { - thriftCLIService = new ThriftHttpCLIService(cliService); + if (transportMode != null && (transportMode.equalsIgnoreCase("http"))) { + return true; } - else { - thriftCLIService = new ThriftBinaryCLIService(cliService); + return false; + } + + /** + * Adds a server instance to ZooKeeper as a znode. + * + * @param hiveConf + * @throws Exception + */ + private void addServerInstanceToZooKeeper(HiveConf hiveConf) throws Exception { + int zooKeeperSessionTimeout = + hiveConf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT); + String zooKeeperEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf); + String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE); + String instanceURI = getServerInstanceURI(hiveConf); + byte[] znodeDataUTF8 = instanceURI.getBytes(Charset.forName("UTF-8")); + zooKeeperClient = + new ZooKeeper(zooKeeperEnsemble, zooKeeperSessionTimeout, + new ZooKeeperHiveHelper.DummyWatcher()); + + // Create the parent znodes recursively; ignore if the parent already exists + try { + ZooKeeperHiveHelper.createPathRecursively(zooKeeperClient, rootNamespace, + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + LOG.info("Created the root name space: " + rootNamespace + " on ZooKeeper for HiveServer2"); + } catch (KeeperException e) { + if (e.code() != KeeperException.Code.NODEEXISTS) { + LOG.fatal("Unable to create HiveServer2 namespace: " + rootNamespace + " on ZooKeeper", e); + throw (e); + } } + // Create a znode under the rootNamespace parent for this instance of the server + // Znode name: server-host:port-versionInfo-sequence + try { + String znodePath = + ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace + + ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + "server-" + instanceURI + "-" + + HiveVersionInfo.getVersion() + "-"; + znodePath = + zooKeeperClient.create(znodePath, znodeDataUTF8, Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL_SEQUENTIAL); + setRegisteredWithZooKeeper(true); + // Set a watch on the znode + if (zooKeeperClient.exists(znodePath, new DeRegisterWatcher()) == null) { + // No node exists, throw exception + throw new Exception("Unable to create znode for this HiveServer2 instance on ZooKeeper."); + } + LOG.info("Created a znode on ZooKeeper for HiveServer2 uri: " + instanceURI); + } catch (KeeperException e) { + LOG.fatal("Unable to create a znode for this server instance", e); + throw new Exception(e); + } + } - addService(thriftCLIService); - super.init(hiveConf); + /** + * The watcher class which sets the de-register flag when the znode corresponding to this server + * instance is deleted. Additionally, it shuts down the server if there are no more active client + * sessions at the time of receiving a 'NodeDeleted' notification from ZooKeeper. + */ + private class DeRegisterWatcher implements Watcher { + public void process(WatchedEvent event) { + if (event.getType().equals(Watcher.Event.EventType.NodeDeleted)) { + HiveServer2.this.setRegisteredWithZooKeeper(false); + // If there are no more active client sessions, stop the server + if (cliService.getSessionManager().getOpenSessionCount() == 0) { + LOG.warn("This instance of HiveServer2 has been removed from the list of server " + + "instances available for dynamic service discovery. " + + "The last client session has ended - will shutdown now."); + HiveServer2.this.stop(); + } + LOG.warn("This HiveServer2 instance is now de-registered from ZooKeeper. " + + "The server will be shut down after the last client sesssion completes."); + } + } + } + + private void removeServerInstanceFromZooKeeper() throws Exception { + setRegisteredWithZooKeeper(false); + zooKeeperClient.close(); + LOG.info("Server instance removed from ZooKeeper."); + } + + public boolean isRegisteredWithZooKeeper() { + return registeredWithZooKeeper; + } + + private void setRegisteredWithZooKeeper(boolean registeredWithZooKeeper) { + this.registeredWithZooKeeper = registeredWithZooKeeper; + } + + private String getServerInstanceURI(HiveConf hiveConf) throws Exception { + if ((thriftCLIService == null) || (thriftCLIService.getServerAddress() == null)) { + throw new Exception("Unable to get the server address; it hasn't been initialized yet."); + } + return thriftCLIService.getServerAddress().getHostName() + ":" + + thriftCLIService.getPortNumber(); } @Override @@ -75,23 +198,32 @@ public class HiveServer2 extends Composi @Override public synchronized void stop() { - super.stop(); - // there should already be an instance of the session pool manager. - // if not, ignoring is fine while stopping the hive server. + LOG.info("Shutting down HiveServer2"); HiveConf hiveConf = this.getHiveConf(); + super.stop(); + // Remove this server instance from ZooKeeper if dynamic service discovery is set + if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) { + try { + removeServerInstanceFromZooKeeper(); + } catch (Exception e) { + LOG.error("Error removing znode for this HiveServer2 instance from ZooKeeper.", e); + } + } + // There should already be an instance of the session pool manager. + // If not, ignoring is fine while stopping HiveServer2. if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS)) { try { TezSessionPoolManager.getInstance().stop(); } catch (Exception e) { - LOG.error("Tez session pool manager stop had an error during stop of hive server"); - e.printStackTrace(); + LOG.error("Tez session pool manager stop had an error during stop of HiveServer2. " + + "Shutting down HiveServer2 anyway.", e); } } } private static void startHiveServer2() throws Throwable { long attempts = 0, maxAttempts = 1; - while(true) { + while (true) { HiveConf hiveConf = new HiveConf(); maxAttempts = hiveConf.getLongVar(HiveConf.ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS); HiveServer2 server = null; @@ -99,6 +231,11 @@ public class HiveServer2 extends Composi server = new HiveServer2(); server.init(hiveConf); server.start(); + // If we're supporting dynamic service discovery, we'll add the service uri for this + // HiveServer2 instance to Zookeeper as a znode. + if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) { + server.addServerInstanceToZooKeeper(hiveConf); + } if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS)) { TezSessionPoolManager sessionPool = TezSessionPoolManager.getInstance(); sessionPool.setupPool(hiveConf); @@ -106,19 +243,19 @@ public class HiveServer2 extends Composi } break; } catch (Throwable throwable) { - if(++attempts >= maxAttempts) { + if (++attempts >= maxAttempts) { throw new Error("Max start attempts " + maxAttempts + " exhausted", throwable); } else { - LOG.warn("Error starting HiveServer2 on attempt " + attempts + - ", will retry in 60 seconds", throwable); + LOG.warn("Error starting HiveServer2 on attempt " + attempts + + ", will retry in 60 seconds", throwable); try { if (server != null) { server.stop(); server = null; } } catch (Exception e) { - LOG.info("Exception caught when calling stop of HiveServer2 before" + - " retrying start", e); + LOG.info( + "Exception caught when calling stop of HiveServer2 before" + " retrying start", e); } try { Thread.sleep(60L * 1000L); @@ -139,14 +276,15 @@ public class HiveServer2 extends Composi System.exit(-1); } - //NOTE: It is critical to do this here so that log4j is reinitialized + // NOTE: It is critical to do this here so that log4j is reinitialized // before any of the other core hive classes are loaded String initLog4jMessage = LogUtils.initHiveLog4j(); LOG.debug(initLog4jMessage); HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG); - //log debug message from "oproc" after log4j initialize properly + // log debug message from "oproc" after log4j initialize properly LOG.debug(oproc.getDebugMessage().toString()); + startHiveServer2(); } catch (LogInitializationException e) { LOG.error("Error initializing log: " + e.getMessage(), e); @@ -156,6 +294,5 @@ public class HiveServer2 extends Composi System.exit(-1); } } - } Modified: hive/branches/cbo/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java?rev=1625461&r1=1625460&r2=1625461&view=diff ============================================================================== --- hive/branches/cbo/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java (original) +++ hive/branches/cbo/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java Wed Sep 17 05:28:43 2014 @@ -27,7 +27,11 @@ import junit.framework.TestCase; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hive.service.cli.*; +import org.apache.hive.service.cli.CLIService; +import org.apache.hive.service.cli.ICLIService; +import org.apache.hive.service.cli.OperationHandle; +import org.apache.hive.service.cli.RowSet; +import org.apache.hive.service.cli.SessionHandle; import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient; import org.junit.After; @@ -83,7 +87,7 @@ public class TestSessionGlobalInitFile e // set up service and client HiveConf hiveConf = new HiveConf(); - hiveConf.setVar(HiveConf.ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION, + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION, initFile.getParentFile().getAbsolutePath()); service = new FakeEmbeddedThriftBinaryCLIService(hiveConf); service.init(new HiveConf());
