[06/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out -- diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out index 1b04155..3844c79 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out @@ -1232,56 +1232,56 @@ LIMIT 50 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc A masked pattern was here -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -51 -51.0 1969-12-31 15:59:43.64 -7196 -1339164819 4992406445232 NULLNULL 7196-14392 -7196 NULLNULL51.06.4051596E8 -5.157308006568995E-5 51 -1.5598627 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -52 -52.0 1969-12-31 15:59:45.978 -7196 -2128720310 7935869315680 NULLNULL 7196-14392 -7196 NULLNULL52.06.4051596E8 -5.258431692972308E-5 52 -1.5298654 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -52 -52.0 1969-12-31 15:59:47.15 -7196 628698169 -2343786774032 NULLNULL 7196-14392 -7196 NULLNULL52.06.4051596E8 -5.258431692972308E-5 52 -1.5298654 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -52 -52.0 1969-12-31 15:59:57.86 -7196 -26309289 98081029392 NULLNULL 7196-14392 -7196 NULLNULL52.06.4051596E8 -5.258431692972308E-5 52 -1.5298654 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -52 -52.0 1969-12-31 15:59:58.479 -7196 -1379694191 5143499944048 NULLNULL 7196-14392 -7196 NULLNULL52.06.4051596E8 -5.258431692972308E-5 52 -1.5298654 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -52 -52.0 1969-12-31 16:00:03.963 -7196 95444104-355815619712 NULLNULL 7196-14392 -7196 NULLNULL52.06.4051596E8 -5.258431692972308E-5 52 -1.5298654 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -52 -52.0 1969-12-31 16:00:04.518 -7196 -1658319459 6182214943152 NULLNULL 7196-14392 -7196 NULLNULL52.06.4051596E8 -5.258431692972308E-5 52 -1.5298654 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -53 -53.0 1969-12-31 15:59:48.882 -7196 -1560660031 5818140595568 NULLNULL 7196-14392 -7196 NULLNULL53.06.4051596E8 -5.359555379375622E-5 53 -1.501 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -53 -53.0 1969-12-31 15:59:57.663 -7196 898472381 -3349505036368 NULLNULL 7196-14392 -7196 NULLNULL53.06.4051596E8 -5.359555379375622E-5 53 -1.501 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -53 -53.0 1969-12-31 16:00:11.36 -7196 -1357789899 5061840743472 NULLNULL 7196-14392 -7196 NULLNULL53.06.4051596E8 -5.359555379375622E-5 53 -1.501 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -54 -54.0 1969-12-31 15:59:53.657 -7196 1476582815 -5504700734320 NULLNULL 7196-14392 -7196 NULLNULL54.06.4051596E8 -5.4606790657789354E-5 54 -1.4732037 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -54 -54.0 1969-12-31 16:00:05.688 -7196 1614836149 -6020109163472 NULLNULL 7196-14392 -7196 NULLNULL54.06.4051596E8 -5.4606790657789354E-5 54 -1.4732037 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -54 -54.0 1969-12-31 16:00:06.484 -7196 1605976008 -5987078557824 NULLNULL 7196-14392 -7196 NULLNULL54.06.4051596E8 -5.4606790657789354E-5 54 -1.4732037 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -54 -54.0 1969-12-31 16:00:11.198 -7196 1650677402 -6153725354656 NULLNULL 7196-14392 -7196 NULLNULL54.06.4051596E8 -5.4606790657789354E-5 54 -1.4732037 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -55 -55.0 1969-12-31 15:59:43.932 -7196 1982381637 -7390318742736 NULLNULL 7196-14392 -7196 NULLNULL55.06.4051596E8 -5.561802752182249E-5 55 -1.4464182 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -55 -55.0 1969-12-31 16:00:01.138 -7196 888532643 -3312449693104 NULLNULL 7196-14392 -7196 NULLNULL55.06.4051596E8 -5.561802752182249E-5 55 -1.4464182 -NULL -7196.0 1969-12-31 15:59:58.174 NULLfalse -55 -55.0 1969-12-31 16:00:13.249 -7196
[04/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/update_where_partitioned.q.out -- diff --git a/ql/src/test/results/clientpositive/update_where_partitioned.q.out b/ql/src/test/results/clientpositive/update_where_partitioned.q.out index 1834e83..ac603b8 100644 --- a/ql/src/test/results/clientpositive/update_where_partitioned.q.out +++ b/ql/src/test/results/clientpositive/update_where_partitioned.q.out @@ -53,11 +53,11 @@ POSTHOOK: Input: default@acid_uwp@ds=tomorrow 762BLoMwUJ51ns6pd tomorrow 762a10E76jX35YwquKCTA tomorrow 762q5y2Vy1 tomorrow -6981 NULLtomorrow 6981 1FNNhmiFLGw425NA13g tomorrow 6981 4KhrrQ0nJ7bMNTvhSCA tomorrow 6981 K630vaVftomorrow 6981 Y5x3JuI3M8jngv5Ntomorrow +6981 YdG61y00526u5 tomorrow PREHOOK: query: update acid_uwp set b = 'fred' where b = 'k17Am8uPHWk02cEf1jet' PREHOOK: type: QUERY PREHOOK: Input: default@acid_uwp @@ -99,8 +99,8 @@ POSTHOOK: Input: default@acid_uwp@ds=tomorrow 762BLoMwUJ51ns6pd tomorrow 762a10E76jX35YwquKCTA tomorrow 762q5y2Vy1 tomorrow -6981 NULLtomorrow 6981 1FNNhmiFLGw425NA13g tomorrow 6981 4KhrrQ0nJ7bMNTvhSCA tomorrow 6981 K630vaVftomorrow 6981 Y5x3JuI3M8jngv5Ntomorrow +6981 YdG61y00526u5 tomorrow http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/vector_case_when_2.q.out -- diff --git a/ql/src/test/results/clientpositive/vector_case_when_2.q.out b/ql/src/test/results/clientpositive/vector_case_when_2.q.out index b1cb3f2..b6acc1d 100644 --- a/ql/src/test/results/clientpositive/vector_case_when_2.q.out +++ b/ql/src/test/results/clientpositive/vector_case_when_2.q.out @@ -235,7 +235,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@timestamps A masked pattern was here ctimestamp1ctimestamp2 ctimestamp2_description ctimestamp2_description_2 ctimestamp2_description_3 field1 field_2 field_3 field_4 field_5 -NULL NULLUnknown NULLNULLNULL2018-03-08 23:04:59 NULL NULLNULL 0004-09-22 18:26:29.51954 0004-09-21 16:23:25.51954 1800s or EarlierOld Old 4 0004-09-22 18:26:29.51954 26 NULL0005-09-22 0528-10-27 08:15:18.941718273 0528-10-26 06:12:14.941718273 1800s or EarlierOld Old 528 2018-03-08 23:04:59 15 NULL 0529-10-27 1319-02-02 16:31:57.7781319-02-01 14:28:53.778 1800s or Earlier Old Old 13191319-02-02 16:31:57.778 31 NULL1320-02-02 @@ -286,6 +285,7 @@ NULLNULLUnknown NULLNULLNULL 2018-03-08 23:04:59 NULLNULLNULL 9075-06-13 16:20:09.218517797 9075-06-12 14:17:05.218517797 Unknown NULL NULL90752018-03-08 23:04:59 20 NULL9075-06-14 9209-11-11 04:08:58.223768453 9209-11-10 02:05:54.223768453 Unknown NULL NULL92092018-03-08 23:04:59 8 NULL9209-11-12 9403-01-09 18:12:33.5479403-01-08 16:09:29.547 Unknown NULLNULL 94032018-03-08 23:04:59 12 NULL9404-01-09 +NULL NULLUnknown NULLNULLNULL2018-03-08 23:04:59 NULL NULLNULL PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT ctimestamp1, @@ -494,7 +494,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@timestamps A masked pattern was here ctimestamp1ctimestamp2 ctimestamp2_description ctimestamp2_description_2 ctimestamp2_description_3 field1 field_2 field_3 field_4 field_5 -NULL NULLUnknown NULLNULLNULL2018-03-08 23:04:59 NULL NULLNULL 0004-09-22 18:26:29.51954 0004-09-21 16:23:25.51954 1800s or EarlierOld Old 4 0004-09-22 18:26:29.51954 26 NULL0005-09-22 0528-10-27 08:15:18.941718273 0528-10-26 06:12:14.941718273 1800s or EarlierOld Old 528 2018-03-08 23:04:59 15 NULL 0529-10-27 1319-02-02 16:31:57.7781319-02-01 14:28:53.778 1800s or Earlier Old Old 13191319-02-02 16:31:57.778 31 NULL1320-02-02 @@ -545,6 +544,7 @@ NULLNULLUnknown NULLNULLNULL 2018-03-08 23:04:59 NULLNULLNULL 9075-06-13 16:20:09.218517797 9075-06-12 14:17:05.218517797 Unknown NULL NULL90752018-03-08 23:04:59 20 NULL9075-06-14 9209-11-11 04:08:58.223768453 9209-11-10 02:05:54.223768453 Unknown NULL NULL92092018-03-08 23:04:59 8 NULL9209-11-12 9403-01-09 18:12:33.5479403-01-08 16:09:29.547 Unknown NULLNULL 94032018-03-08 23:04:59 12 NULL9404-01-09 +NULL NULLUnknown NULLNULLNULL2018-03-08 23:04:59 NULL NULLNULL
[17/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out index 1087902..e4f2452 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out @@ -155,7 +155,7 @@ STAGE PLANS: Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) - null sort order: a + null sort order: z sort order: + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE tag: -1 @@ -541,7 +541,7 @@ STAGE PLANS: Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) - null sort order: a + null sort order: z sort order: + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE tag: -1 @@ -818,7 +818,7 @@ STAGE PLANS: Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) - null sort order: a + null sort order: z sort order: + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE tag: -1 @@ -1095,7 +1095,7 @@ STAGE PLANS: Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) - null sort order: a + null sort order: z sort order: + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE tag: -1 http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/subquery_in.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out index 21801a3..5a9021b 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -345,7 +345,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col5 ASC NULLS FIRST +order by: _col5 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -528,7 +528,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col5 ASC NULLS FIRST +order by: _col5 ASC NULLS LAST partition by: _col2 raw input shape: window functions: http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out b/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out index ab28bba..1b25b5c 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out @@ -1508,7 +1508,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col5 ASC NULLS FIRST +order by: _col5 ASC NULLS LAST partition by: _col2 raw input shape: window functions: http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
[02/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/vector_outer_join4.q.out -- diff --git a/ql/src/test/results/clientpositive/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/vector_outer_join4.q.out index 2826698..adbb759 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join4.q.out @@ -130,16 +130,16 @@ POSTHOOK: query: select * from small_alltypesorc3b POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc3b A masked pattern was here -NULL NULL-1015272448 -1887561756 NULLNULLjTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:45.854 false false -NULL NULL-609074876 -1887561756 NULLNULLEcM71 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:55.061 truefalse -NULL NULL-700300206 -1887561756 NULLNULLkdqQE010 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:58.384 false false -NULL NULL-726473298 1864027286 NULLNULL OFy1a1xf37f75b5N4KWs6gw7lv2WYd66P NULL1969-12-31 16:00:11.799 truetrue -NULL NULL-738747840 -1645852809 NULLNULL vmAT10eeE47fgH20pLi xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:11.55 truefalse -NULL NULL-838810013 1864027286 NULLNULLN016jPED08o 4KWs6gw7lv2WYd66P NULL1969-12-31 15:59:44.252 false true -NULL NULL-850295959 -1887561756 NULLNULLWMIgGA73 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:00.348 false false -NULL NULL-886426182 -1887561756 NULLNULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:04.472 truefalse -NULL NULL-89947 -1645852809 NULLNULL73xdw4X xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:07.395 false false -NULL NULL-971543377 -1645852809 NULLNULLuN803aW xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:05.43 false false +NULL -16269 -378213344 -1645852809 NULL-16269.0 sOdj1Tmvbl03f xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:15.867 false false +NULL -16274 -671342269 -1645852809 NULL-16274.0 3DE7EQo4KyT0hS xH7445Rals48VOulSyR5F NULL1969-12-31 15:59:51.469 false false +NULL -16296 -146635689 -1645852809 NULL-16296.0 r251rbt884txX2MNq4MM14 xH7445Rals48VOulSyR5F NULL1969-12-31 15:59:44.809 false false +NULL -16296 593429004 -1887561756 NULL-16296.0 dhDYJ076SFcC4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:47.422 false false +NULL -16300 -860437234 -1645852809 NULL-16300.0 Fb2W1r24opqN8m6571p xH7445Rals48VOulSyR5F NULL1969-12-31 15:59:45.815 truefalse +NULL -16306 384405526 -1645852809 NULL-16306.0b5SoK8 xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:11.105 truefalse +NULL -16307 559926362 -1645852809 NULL-16307.0 nA8bdtWfPPQyP2hL5 xH7445Rals48VOulSyR5F NULL1969-12-31 15:59:58.072 false false +NULL -16309 -826497289 -1645852809 NULL-16309.0 54o058c3mK6ewOQ5xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:10.761 false false +NULL -16310 206154150 1864027286 NULL-16310.05Hy1y6 4KWs6gw7lv2WYd66P NULL1969-12-31 16:00:00.821 false true +NULL -16379 -894716315 1864027286 NULL-16379.0 2ArdYqML3654nUjGJk3 4KWs6gw7lv2WYd66P NULL1969-12-31 15:59:47.059 truetrue PREHOOK: query: select * from small_alltypesorc4b PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc4b @@ -236,16 +236,16 @@ POSTHOOK: Input: default@small_alltypesorc_b -64-7196 NULL658026952 -64.0 -7196.0 NULL4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULLtrue -64-8080 528534767 NULL-64.0 -8080.0 cvLH6Eat2yFsyy7p NULL1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 trueNULL -64-9842 253665376 NULL-64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 trueNULL -NULL NULL-1015272448 -1887561756 NULLNULLjTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:45.854 false false -NULL NULL-609074876 -1887561756 NULLNULLEcM71 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:55.061 truefalse -NULL NULL-700300206 -1887561756 NULLNULLkdqQE010 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:58.384 false false -NULL NULL-726473298 1864027286 NULLNULL OFy1a1xf37f75b5N4KWs6gw7lv2WYd66P NULL1969-12-31 16:00:11.799
[18/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index b8ea5cf..42c2f5b 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -207,7 +207,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n2 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 16675 + HDFS_BYTES_READ: 16681 HDFS_BYTES_WRITTEN: 104 HDFS_READ_OPS: 7 HDFS_LARGE_READ_OPS: 0 @@ -267,7 +267,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n2 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1055 + HDFS_BYTES_READ: 1047 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 5 HDFS_LARGE_READ_OPS: 0 @@ -288,9 +288,9 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1048576 - ALLOCATED_USED_BYTES: 2732 + ALLOCATED_USED_BYTES: 2731 CACHE_HIT_BYTES: 24 - CACHE_MISS_BYTES: 1055 + CACHE_MISS_BYTES: 1047 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -327,7 +327,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 22 RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -365,7 +365,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 16 RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -403,7 +403,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 18 RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -441,7 +441,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 1 RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -479,7 +479,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 32 RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -517,7 +517,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 32 RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -555,7 +555,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 1697 RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -593,7 +593,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 12 RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -631,7 +631,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 1713 RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -657,7 +657,7 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 @@ -667,14 +667,14 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_MAP_0: 0 RECORDS_OUT_OPERATOR_RS_3: 6 RECORDS_OUT_OPERATOR_SEL_2: 6 - RECORDS_OUT_OPERATOR_TS_0: 1000 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 + ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 @@ -707,7 +707,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 50 RECORDS_OUT_OPERATOR_TS_0: 1100 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1079 + CACHE_HIT_BYTES: 1071 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -745,7 +745,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 318
[13/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/vectorization_7.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/vectorization_7.q.out b/ql/src/test/results/clientpositive/llap/vectorization_7.q.out index 008068e..c18133b 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_7.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_7.q.out @@ -126,7 +126,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: aaa +reduceColumnNullOrder: zzz reduceColumnSortOrder: +++ allNative: false usesVectorUDFAdaptor: false @@ -224,31 +224,31 @@ LIMIT 25 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc A masked pattern was here -NULL -2118149242 -7196 56 1969-12-31 15:59:50.462 NULL -4236298484 0 7196-56 -39 -15242201945432 NULL-56 0 -NULL -2121399625 -7196 27 1969-12-31 15:59:50.046 NULL -4242799250 0 7196-27 -10 -15265591701500 NULL-27 0 -NULL -2124802690 -7196 -6 1969-12-31 15:59:57.92 NULL -4249605380 0 71966 23 -15290080157240 NULL6 0 -NULL -2128720310 -7196 -52 1969-12-31 15:59:45.978 NULL -4257440620 0 719652 69 -15318271350760 NULL52 0 -NULL -2132232110 -20060 1969-12-31 15:59:47.019 NULL -4264464220 -200200 -60 -43 -426446422000 NULL-60 0 -NULL -2132536965 -7196 9 1969-12-31 15:59:46 NULL -4265073930 0 7196-9 8 -15345736000140 NULL-9 0 -NULL -2135141157 -7196 50 1969-12-31 15:59:50.192 NULL -4270282314 0 7196-50 -33 -15364475765772 NULL-50 0 -NULL -2137537679 -7196 -25 1969-12-31 15:59:50.136 NULL -4275075358 0 719625 42 -15381721138084 NULL25 0 -NULL -2145481991 -7196 56 1969-12-31 15:59:55.667 NULL -4290963982 0 7196-56 -39 -1543407236 NULL-56 0 -NULL NULL-200-36 1969-12-31 15:59:57.241 NULLNULL-200 200 36 53 NULLNULL36 0 -NULL NULL-200-43 1969-12-31 15:59:53.783 NULLNULL-200 200 43 60 NULLNULL43 0 -NULL NULL-200-58 1969-12-31 15:59:51.115 NULLNULL-200 200 58 75 NULLNULL58 0 -NULL NULL-20022 1969-12-31 15:59:50.109 NULLNULL-200 200 -22 -5 NULLNULL-22 0 -NULL NULL-2003 1969-12-31 15:59:50.489 NULLNULL-200 200 -3 14 NULLNULL-3 0 -NULL NULL-20043 1969-12-31 15:59:57.003 NULLNULL-200 200 -43 -26 NULLNULL-43 0 -NULL NULL-20053 1969-12-31 15:59:49.46 NULLNULL-200 200 -53 -36 NULLNULL-53 0 -NULL NULL-2009 1969-12-31 15:59:44.108 NULLNULL-200 200 -9 8 NULLNULL-9 0 -NULL NULL-7196 -38 1969-12-31 15:59:53.503 NULLNULL0 719638 55 NULLNULL38 0 -NULL NULL-7196 -49 1969-12-31 15:59:51.009 NULLNULL0 719649 66 NULLNULL49 0 -NULL NULL-7196 -49 1969-12-31 15:59:52.052 NULLNULL0 719649 66 NULLNULL49 0 -NULL NULL-7196 -50 1969-12-31 15:59:52.424 NULLNULL0 719650 67 NULLNULL50 0 -NULL NULL-7196 -61 1969-12-31 15:59:44.823 NULLNULL0 719661 78 NULLNULL61 0 -NULL NULL-7196 1 1969-12-31 15:59:48.361 NULLNULL0 7196-1 16 NULLNULL-1 0 -NULL NULL-7196 14 1969-12-31 15:59:50.291 NULLNULL0 7196-14 3 NULLNULL-14 0 -NULL NULL-7196 22 1969-12-31 15:59:52.699 NULLNULL0 7196-22 -5 NULLNULL-22 0 +true NULL-15892 29 1969-12-31 15:59:57.937 821UdmGbkEf4j NULL -21515892 -29 -12 NULL171 -29 0 +true NULL-15899 50 1969-12-31 15:59:46.926 821UdmGbkEf4j NULL -22215899 -50 -33 NULL10210 -50 0 +true NULL-15903 -2 1969-12-31 15:59:46.371 cvLH6Eat2yFsyy7p NULL-22615903 2 19
[11/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out -- diff --git a/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out b/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out index 6459e29..66af40c 100644 --- a/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out @@ -290,46 +290,46 @@ LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesparquet A masked pattern was here -NULL -55 1969-12-31 16:00:11.38 -55.0 NULL55 -55 0 -55.0 -0.055.0-4375.415 0.0 55.00.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55 -NULL -55 1969-12-31 16:00:11.751 -55.0 NULL55 -55 0 -55.0 -0.055.0-4375.415 0.0 55.00.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55 -NULL -56 1969-12-31 16:00:13.602 -56.0 NULL56 -56 0 -56.0 -0.056.0-4454.9683 0.0 56.00.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56 -NULL -56 1969-12-31 16:00:13.958 -56.0 NULL56 -56 0 -56.0 -0.056.0-4454.9683 0.0 56.00.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56 -NULL -56 1969-12-31 16:00:15.038 -56.0 NULL56 -56 0 -56.0 -0.056.0-4454.9683 0.0 56.00.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56 -NULL -57 1969-12-31 16:00:11.451 -57.0 NULL57 -57 0 -57.0 -0.057.0-4534.521 0.0 57.00.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 -NULL -57 1969-12-31 16:00:11.883 -57.0 NULL57 -57 0 -57.0 -0.057.0-4534.521 0.0 57.00.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 -NULL -57 1969-12-31 16:00:12.626 -57.0 NULL57 -57 0 -57.0 -0.057.0-4534.521 0.0 57.00.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 -NULL -57 1969-12-31 16:00:13.578 -57.0 NULL57 -57 0 -57.0 -0.057.0-4534.521 0.0 57.00.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 -NULL -57 1969-12-31 16:00:15.39 -57.0 NULL57 -57 0 -57.0 -0.057.0-4534.521 0.0 57.00.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 -NULL -58 1969-12-31 16:00:12.065 -58.0 NULL58 -58 0 -58.0 -0.058.0-4614.074 0.0 58.00.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 -NULL -58 1969-12-31 16:00:12.683 -58.0 NULL58 -58 0 -58.0 -0.058.0-4614.074 0.0 58.00.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 -NULL -58 1969-12-31 16:00:12.948 -58.0 NULL58 -58 0 -58.0 -0.058.0-4614.074 0.0 58.00.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 -NULL -58 1969-12-31 16:00:14.066 -58.0 NULL58 -58 0 -58.0 -0.058.0-4614.074 0.0 58.00.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 -NULL -58 1969-12-31 16:00:15.658 -58.0 NULL58 -58 0 -58.0 -0.058.0-4614.074 0.0 58.00.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 -NULL -59 1969-12-31 16:00:12.008 -59.0 NULL59 -59 0 -59.0 -0.059.0-4693.627 0.0 59.00.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 -NULL -59 1969-12-31 16:00:13.15 -59.0 NULL59 -59 0 -59.0 -0.059.0-4693.627 0.0 59.00.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 -NULL -59 1969-12-31 16:00:13.625 -59.0 NULL59 -59 0 -59.0 -0.059.0-4693.627 0.0 59.00.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 -NULL -59 1969-12-31 16:00:15.296 -59.0 NULL59 -59 0 -59.0 -0.059.0-4693.627 0.0 59.00.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 -NULL -59 1969-12-31 16:00:15.861 -59.0 NULL59 -59 0 -59.0 -0.059.0-4693.627 0.0 59.00.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 -NULL -60 1969-12-31 16:00:11.504 -60.0 NULL60 -60 0 -60.0 -0.060.0-4773.180.0 60.00.0 -10.175 -60.0 0.438 -60.0 0.0 -60 -NULL -60 1969-12-31
[10/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/perf/tez/query49.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/tez/query49.q.out
b/ql/src/test/results/clientpositive/perf/tez/query49.q.out
index 6f642ef..5c1889c 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query49.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query49.q.out
@@ -307,7 +307,7 @@ Stage-0
Filter Operator [FIL_347] (rows=8604378 width=88)
predicate:((_col0 <= 10) or (rank_window_1 <=
10))
PTF Operator [PTF_346] (rows=12906568 width=88)
-Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col4 AS
decimal(15,4)) / CAST( _col5 AS decimal(15,4))) ASC NULLS FIRST","partition
by:":"0"}]
+Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col4 AS
decimal(15,4)) / CAST( _col5 AS decimal(15,4))) ASC NULLS LAST","partition
by:":"0"}]
Select Operator [SEL_345] (rows=12906568
width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
<-Reducer 23 [SIMPLE_EDGE] vectorized
@@ -316,7 +316,7 @@ Stage-0
Select Operator [SEL_343] (rows=12906568
width=88)
Output:["rank_window_0","_col0","_col1","_col2","_col3","_col4"]
PTF Operator [PTF_342] (rows=12906568
width=88)
-Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col1 AS
decimal(15,4)) / CAST( _col2 AS decimal(15,4))) ASC NULLS FIRST","partition
by:":"0"}]
+Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col1 AS
decimal(15,4)) / CAST( _col2 AS decimal(15,4))) ASC NULLS LAST","partition
by:":"0"}]
Select Operator [SEL_341]
(rows=12906568 width=88)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Reducer 22 [SIMPLE_EDGE] vectorized
@@ -398,7 +398,7 @@ Stage-0
Filter Operator [FIL_327] (rows=4302070
width=135)
predicate:((_col0 <= 10) or
(rank_window_1 <= 10))
PTF Operator [PTF_326] (rows=6453105
width=135)
- Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col4 AS
decimal(15,4)) / CAST( _col5 AS decimal(15,4))) ASC NULLS FIRST","partition
by:":"0"}]
+ Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col4 AS
decimal(15,4)) / CAST( _col5 AS decimal(15,4))) ASC NULLS LAST","partition
by:":"0"}]
Select Operator [SEL_325]
(rows=6453105 width=135)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
<-Reducer 17 [SIMPLE_EDGE] vectorized
@@ -407,7 +407,7 @@ Stage-0
Select Operator [SEL_323]
(rows=6453105 width=135)
Output:["rank_window_0","_col0","_col1","_col2","_col3","_col4"]
PTF Operator [PTF_322]
(rows=6453105 width=135)
- Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col1 AS
decimal(15,4)) / CAST( _col2 AS decimal(15,4))) ASC NULLS FIRST","partition
by:":"0"}]
+ Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col1 AS
decimal(15,4)) / CAST( _col2 AS decimal(15,4))) ASC NULLS LAST","partition
by:":"0"}]
Select Operator [SEL_321]
(rows=6453105 width=135)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Reducer 16 [SIMPLE_EDGE]
vectorized
@@ -472,7 +472,7 @@ Stage-0
Filter Operator [FIL_296] (rows=2151150
width=135)
predicate:((_col0 <= 10) or
(rank_window_1 <= 10))
PTF Operator [PTF_295] (rows=3226726
width=135)
- Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col4 AS
decimal(15,4)) /
[07/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out -- diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out index 9be72ea..70f7401 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out @@ -130,16 +130,16 @@ POSTHOOK: query: select * from small_alltypesorc3b POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc3b POSTHOOK: Output: hdfs://### HDFS PATH ### -NULL NULL-1015272448 -1887561756 NULLNULLjTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:45.854 false false -NULL NULL-609074876 -1887561756 NULLNULLEcM71 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:55.061 truefalse -NULL NULL-700300206 -1887561756 NULLNULLkdqQE010 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:58.384 false false -NULL NULL-726473298 1864027286 NULLNULL OFy1a1xf37f75b5N4KWs6gw7lv2WYd66P NULL1969-12-31 16:00:11.799 truetrue -NULL NULL-738747840 -1645852809 NULLNULL vmAT10eeE47fgH20pLi xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:11.55 truefalse -NULL NULL-838810013 1864027286 NULLNULLN016jPED08o 4KWs6gw7lv2WYd66P NULL1969-12-31 15:59:44.252 false true -NULL NULL-850295959 -1887561756 NULLNULLWMIgGA73 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:00.348 false false -NULL NULL-886426182 -1887561756 NULLNULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:04.472 truefalse -NULL NULL-89947 -1645852809 NULLNULL73xdw4X xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:07.395 false false -NULL NULL-971543377 -1645852809 NULLNULLuN803aW xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:05.43 false false +NULL -16269 -378213344 -1645852809 NULL-16269.0 sOdj1Tmvbl03f xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:15.867 false false +NULL -16274 -671342269 -1645852809 NULL-16274.0 3DE7EQo4KyT0hS xH7445Rals48VOulSyR5F NULL1969-12-31 15:59:51.469 false false +NULL -16296 -146635689 -1645852809 NULL-16296.0 r251rbt884txX2MNq4MM14 xH7445Rals48VOulSyR5F NULL1969-12-31 15:59:44.809 false false +NULL -16296 593429004 -1887561756 NULL-16296.0 dhDYJ076SFcC4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:47.422 false false +NULL -16300 -860437234 -1645852809 NULL-16300.0 Fb2W1r24opqN8m6571p xH7445Rals48VOulSyR5F NULL1969-12-31 15:59:45.815 truefalse +NULL -16306 384405526 -1645852809 NULL-16306.0b5SoK8 xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:11.105 truefalse +NULL -16307 559926362 -1645852809 NULL-16307.0 nA8bdtWfPPQyP2hL5 xH7445Rals48VOulSyR5F NULL1969-12-31 15:59:58.072 false false +NULL -16309 -826497289 -1645852809 NULL-16309.0 54o058c3mK6ewOQ5xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:10.761 false false +NULL -16310 206154150 1864027286 NULL-16310.05Hy1y6 4KWs6gw7lv2WYd66P NULL1969-12-31 16:00:00.821 false true +NULL -16379 -894716315 1864027286 NULL-16379.0 2ArdYqML3654nUjGJk3 4KWs6gw7lv2WYd66P NULL1969-12-31 15:59:47.059 truetrue PREHOOK: query: select * from small_alltypesorc4b PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc4b @@ -236,16 +236,16 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### -64-7196 NULL658026952 -64.0 -7196.0 NULL4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULLtrue -64-8080 528534767 NULL-64.0 -8080.0 cvLH6Eat2yFsyy7p NULL1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 trueNULL -64-9842 253665376 NULL-64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 trueNULL -NULL NULL-1015272448 -1887561756 NULLNULLjTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:45.854 false false -NULL NULL-609074876 -1887561756 NULLNULLEcM71 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:55.061 truefalse -NULL NULL-700300206 -1887561756 NULLNULLkdqQE010 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:58.384 false false -NULL NULL-726473298 1864027286 NULLNULL OFy1a1xf37f75b5N4KWs6gw7lv2WYd66P
[16/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out index 5e7e8ca..e9e9290 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out @@ -59,6 +59,37 @@ POSTHOOK: query: SELECT * FROM DECIMAL_PRECISION ORDER BY `dec` POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_precision A masked pattern was here +0.00 +0.00 +0.00 +0.00 +0.00 +0.1234567890 +0.1234567890 +1.2345678901 +1.2345678901 +1.2345678901 +12.3456789012 +12.3456789012 +12.3456789012 +123.4567890123 +123.4567890123 +123.4567890123 +1234.5678901235 +1234.5678901235 +1234.5678901235 +12345.6789012346 +12345.6789012346 +123456.7890123456 +123456.7890123457 +1234567.8901234560 +1234567.8901234568 +12345678.9012345600 +12345678.9012345679 +123456789.0123456000 +123456789.0123456789 +1234567890.123456 +1234567890.1234567890 NULL NULL NULL @@ -103,37 +134,6 @@ NULL NULL NULL NULL -0.00 -0.00 -0.00 -0.00 -0.00 -0.1234567890 -0.1234567890 -1.2345678901 -1.2345678901 -1.2345678901 -12.3456789012 -12.3456789012 -12.3456789012 -123.4567890123 -123.4567890123 -123.4567890123 -1234.5678901235 -1234.5678901235 -1234.5678901235 -12345.6789012346 -12345.6789012346 -123456.7890123456 -123456.7890123457 -1234567.8901234560 -1234567.8901234568 -12345678.9012345600 -12345678.9012345679 -123456789.0123456000 -123456789.0123456789 -1234567890.123456 -1234567890.1234567890 PREHOOK: query: SELECT `dec`, `dec` + 1, `dec` - 1 FROM DECIMAL_PRECISION ORDER BY `dec` PREHOOK: type: QUERY PREHOOK: Input: default@decimal_precision @@ -142,50 +142,6 @@ POSTHOOK: query: SELECT `dec`, `dec` + 1, `dec` - 1 FROM DECIMAL_PRECISION ORDER POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_precision A masked pattern was here -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL -NULL NULLNULL 0.00 1.00-1.00 0.00 1.00-1.00 0.00 1.00-1.00 @@ -217,14 +173,6 @@ NULL NULLNULL 123456789.0123456789 123456790.0123456789123456788.0123456789 1234567890.123456 1234567891.123456 1234567889.123456 1234567890.1234567890 1234567891.1234567890 1234567889.1234567890 -PREHOOK: query: SELECT `dec`, `dec` * 2, `dec` / 3 FROM DECIMAL_PRECISION ORDER BY `dec` -PREHOOK: type: QUERY -PREHOOK: Input: default@decimal_precision - A masked pattern was here -POSTHOOK: query: SELECT `dec`, `dec` * 2, `dec` / 3 FROM DECIMAL_PRECISION ORDER BY `dec` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@decimal_precision - A masked pattern was here NULL NULLNULL NULL NULLNULL NULL NULLNULL @@ -269,6 +217,14 @@ NULL NULLNULL NULL NULLNULL NULL NULLNULL NULL NULLNULL +PREHOOK: query: SELECT `dec`, `dec` * 2, `dec` / 3 FROM DECIMAL_PRECISION ORDER BY `dec` +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision + A masked pattern was here +POSTHOOK: query: SELECT `dec`, `dec` * 2, `dec` / 3 FROM DECIMAL_PRECISION ORDER BY `dec` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision + A masked pattern was here 0.00 0.000. 0.00 0.000. 0.00 0.000. @@ -300,6 +256,50 @@ NULL NULLNULL 123456789.0123456789 246913578.024691357841152263.004115226300 1234567890.123456 2469135780.246912 411522630.04115200 1234567890.1234567890 2469135780.2469135780 411522630.041152263000 +NULL NULLNULL +NULL NULLNULL +NULL NULLNULL +NULL NULLNULL +NULL NULLNULL +NULL
[01/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
Repository: hive Updated Branches: refs/heads/master 35f86c749 -> 499d01a38 http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out -- diff --git a/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out b/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out index 4da63b1..fc7ad07 100644 --- a/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out +++ b/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out @@ -253,7 +253,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc_string -Statistics: Num rows: 52 Data size: 3179 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 52 Data size: 8979 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true Select Operator @@ -264,7 +264,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 3, 13, 14, 15, 16, 17] selectExpressions: VectorUDFUnixTimeStampTimestamp(col 1:timestamp) -> 5:bigint, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 6:int, VectorUDFMonthTimestamp(col 1:timestamp, field MONTH) -> 7:int, VectorUDFDayOfMonthTimestamp(col 1:timestamp, field DAY_OF_MONTH) -> 8:int, VectorUDFWeekOfYearTimestamp(col 1:timestamp, field WEEK_OF_YEAR) -> 9:int, VectorUDFHourTimestamp(col 1:timestamp, field HOUR_OF_DAY) -> 10:int, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 11:int, VectorUDFSecondTimestamp(col 1:timestamp, field SECOND) -> 12:int, IfExprTimestampColumnScalar(col 0:boolean, col 1:timestamp, val 1319-01-25 08:31:57.778) -> 13:timestamp, IfExprTimestampScalarColumn(col 0:boolean, val 2000-12-18 00:42:30.0005, col 1:timestamp) -> 14:timestamp, IfExprTimestampColumnColumn(col 0:boolean, col 1:timestampcol 3:timestamp) -> 15:timestamp, IfExprColumnNull(col 0:boolean, col 1:timestamp, null)(children: col 0:boolean, col 1:timestamp) -> 16:timestam p, IfExprNullColumn(col 0:boolean, null, col 3)(children: col 0:boolean, col 3:timestamp) -> 17:timestamp - Statistics: Num rows: 52 Data size: 3179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 52 Data size: 8979 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: bigint) sort order: + @@ -273,7 +273,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false -Statistics: Num rows: 52 Data size: 3179 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 52 Data size: 8979 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: boolean), _col10 (type: timestamp), _col11 (type: timestamp), _col12 (type: timestamp), _col13 (type: timestamp), _col14 (type: timestamp), _col15 (type: timestamp), _col16 (type: timestamp) Execution mode: vectorized Map Vectorization: @@ -293,10 +293,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: bigint), VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: int), VALUE._col7 (type: boolean), VALUE._col8 (type: timestamp), VALUE._col9 (type: timestamp), VALUE._col10 (type: timestamp), VALUE._col11 (type: timestamp), VALUE._col12 (type: timestamp), VALUE._col13 (type: timestamp), VALUE._col14 (type: timestamp) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 52 Data size: 3179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 52 Data size: 8979 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false -Statistics: Num rows: 52 Data size: 3179 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 52 Data size: 8979 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format:
[03/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/vector_outer_join3.q.out -- diff --git a/ql/src/test/results/clientpositive/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/vector_outer_join3.q.out index 1f1c131..6cae357 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join3.q.out @@ -94,11 +94,11 @@ POSTHOOK: query: select * from small_alltypesorc1a_n1 POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc1a_n1 A masked pattern was here -NULL NULL-1015272448 -1887561756 NULLNULLjTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:45.854 false false -NULL NULL-850295959 -1887561756 NULLNULLWMIgGA73 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:00.348 false false -NULL NULL-886426182 -1887561756 NULLNULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:04.472 truefalse -NULL NULL-89947 -1645852809 NULLNULL73xdw4X xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:07.395 false false -NULL NULL-971543377 -1645852809 NULLNULLuN803aW xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:05.43 false false +-64-10462 626923679 NULL-64.0 -10462.0821UdmGbkEf4j NULL1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 trueNULL +-64-15920 528534767 NULL-64.0 -15920.0 cvLH6Eat2yFsyy7pNULL1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 trueNULL +-64-6907 253665376 NULL-64.0 -6907.0 1cGVWH7n1QU NULL NULL1969-12-31 15:59:53.66 trueNULL +-64-8080 528534767 NULL-64.0 -8080.0 cvLH6Eat2yFsyy7p NULL1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 trueNULL +-64-9842 253665376 NULL-64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 trueNULL PREHOOK: query: select * from small_alltypesorc2a_n1 PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc2a_n1 @@ -216,16 +216,16 @@ POSTHOOK: Input: default@small_alltypesorc_a_n1 -51NULLNULL-1874052220 -51.0 NULLc61B47I604gymFJ sjWQS78 1969-12-31 16:00:08.451 NULLfalse false -51NULLNULL-1927203921 -51.0 NULL45ja5suO42S0I0 1969-12-31 16:00:08.451 NULLtruetrue -51NULLNULL-1970551565 -51.0 NULLr2uhJH3 loXMWyrHjVeK 1969-12-31 16:00:08.451 NULLfalse false +-64-10462 626923679 NULL-64.0 -10462.0821UdmGbkEf4j NULL1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 trueNULL +-64-15920 528534767 NULL-64.0 -15920.0 cvLH6Eat2yFsyy7pNULL1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 trueNULL +-64-6907 253665376 NULL-64.0 -6907.0 1cGVWH7n1QU NULL NULL1969-12-31 15:59:53.66 trueNULL -64-7196 NULL-1615920595 -64.0 -7196.0 NULLX5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULLfalse -64-7196 NULL-1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false -64-7196 NULL-527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I721969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true -64-7196 NULL406535485 -64.0 -7196.0 NULLE011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULLfalse -64-7196 NULL658026952 -64.0 -7196.0 NULL4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULLtrue -NULL NULL-1015272448 -1887561756 NULLNULLjTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:45.854 false false -NULL NULL-850295959 -1887561756 NULLNULLWMIgGA73 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:00.348 false false -NULL NULL-886426182 -1887561756 NULLNULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:04.472 truefalse -NULL NULL-89947 -1645852809 NULLNULL73xdw4X xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:07.395 false false -NULL NULL-971543377 -1645852809 NULLNULLuN803aW xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:05.43 false false +-64-8080 528534767 NULL-64.0 -8080.0 cvLH6Eat2yFsyy7p NULL1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 trueNULL +-64-9842 253665376 NULL-64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 trueNULL PREHOOK: query: explain vectorization detail formatted select count(*) from (select c.cstring1
[15/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out index af69747..a975d8a 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out @@ -94,11 +94,11 @@ POSTHOOK: query: select * from small_alltypesorc1a_n1 POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc1a_n1 A masked pattern was here -NULL NULL-1015272448 -1887561756 NULLNULLjTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:45.854 false false -NULL NULL-850295959 -1887561756 NULLNULLWMIgGA73 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:00.348 false false -NULL NULL-886426182 -1887561756 NULLNULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:04.472 truefalse -NULL NULL-89947 -1645852809 NULLNULL73xdw4X xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:07.395 false false -NULL NULL-971543377 -1645852809 NULLNULLuN803aW xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:05.43 false false +-64-10462 626923679 NULL-64.0 -10462.0821UdmGbkEf4j NULL1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 trueNULL +-64-15920 528534767 NULL-64.0 -15920.0 cvLH6Eat2yFsyy7pNULL1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 trueNULL +-64-6907 253665376 NULL-64.0 -6907.0 1cGVWH7n1QU NULL NULL1969-12-31 15:59:53.66 trueNULL +-64-8080 528534767 NULL-64.0 -8080.0 cvLH6Eat2yFsyy7p NULL1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 trueNULL +-64-9842 253665376 NULL-64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 trueNULL PREHOOK: query: select * from small_alltypesorc2a_n1 PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc2a_n1 @@ -216,16 +216,16 @@ POSTHOOK: Input: default@small_alltypesorc_a_n1 -51NULLNULL-1874052220 -51.0 NULLc61B47I604gymFJ sjWQS78 1969-12-31 16:00:08.451 NULLfalse false -51NULLNULL-1927203921 -51.0 NULL45ja5suO42S0I0 1969-12-31 16:00:08.451 NULLtruetrue -51NULLNULL-1970551565 -51.0 NULLr2uhJH3 loXMWyrHjVeK 1969-12-31 16:00:08.451 NULLfalse false +-64-10462 626923679 NULL-64.0 -10462.0821UdmGbkEf4j NULL1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 trueNULL +-64-15920 528534767 NULL-64.0 -15920.0 cvLH6Eat2yFsyy7pNULL1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 trueNULL +-64-6907 253665376 NULL-64.0 -6907.0 1cGVWH7n1QU NULL NULL1969-12-31 15:59:53.66 trueNULL -64-7196 NULL-1615920595 -64.0 -7196.0 NULLX5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULLfalse -64-7196 NULL-1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false -64-7196 NULL-527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I721969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true -64-7196 NULL406535485 -64.0 -7196.0 NULLE011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULLfalse -64-7196 NULL658026952 -64.0 -7196.0 NULL4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULLtrue -NULL NULL-1015272448 -1887561756 NULLNULLjTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 15:59:45.854 false false -NULL NULL-850295959 -1887561756 NULLNULLWMIgGA73 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:00.348 false false -NULL NULL-886426182 -1887561756 NULLNULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL1969-12-31 16:00:04.472 truefalse -NULL NULL-89947 -1645852809 NULLNULL73xdw4X xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:07.395 false false -NULL NULL-971543377 -1645852809 NULLNULLuN803aW xH7445Rals48VOulSyR5F NULL1969-12-31 16:00:05.43 false false +-64-8080 528534767 NULL-64.0 -8080.0 cvLH6Eat2yFsyy7p NULL1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 trueNULL +-64-9842 253665376 NULL-64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 trueNULL PREHOOK: query: explain vectorization detail formatted select count(*)
[20/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez) Signed-off-by: Teddy Choi Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/499d01a3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/499d01a3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/499d01a3 Branch: refs/heads/master Commit: 499d01a38d34fc1db51a04b4e9d24281ef4fe4d3 Parents: 35f86c7 Author: Teddy Choi Authored: Fri Sep 14 11:48:20 2018 +0900 Committer: Teddy Choi Committed: Fri Sep 14 11:48:21 2018 +0900 -- .../org/apache/hadoop/hive/conf/HiveConf.java |2 + .../write_final_output_blobstore.q.out |4 +- .../hive/ql/parse/BaseSemanticAnalyzer.java |9 +- .../hadoop/hive/ql/parse/CalcitePlanner.java|4 +- .../apache/hadoop/hive/ql/parse/HiveParser.g| 14 +- .../hadoop/hive/ql/plan/PTFDeserializer.java|4 +- .../hive/ql/udf/generic/GenericUDAFAverage.java | 16 +- .../ql/udf/generic/GenericUDAFEvaluator.java| 11 +- .../hive/ql/udf/generic/GenericUDAFSum.java | 18 +- .../hive/ql/udf/ptf/BasePartitionEvaluator.java | 49 +- .../hive/ql/udf/ptf/TableFunctionEvaluator.java |9 + .../hive/ql/udf/ptf/TableFunctionResolver.java |5 +- .../hive/ql/udf/ptf/ValueBoundaryScanner.java | 93 +- .../hive/ql/udf/ptf/WindowingTableFunction.java |4 +- .../udf/generic/TestGenericUDAFEvaluator.java |9 +- .../clientpositive/beeline/smb_mapjoin_13.q.out |4 +- ...names_with_leading_and_trailing_spaces.q.out |2 +- .../clientpositive/correlationoptimizer14.q.out | 37 - .../results/clientpositive/ctas_colname.q.out |4 +- .../test/results/clientpositive/decimal_3.q.out |8 +- .../test/results/clientpositive/decimal_4.q.out |4 +- .../test/results/clientpositive/decimal_5.q.out |8 +- .../test/results/clientpositive/decimal_6.q.out | 12 +- .../clientpositive/decimal_precision.q.out | 286 ++--- .../results/clientpositive/decimal_serde.q.out |4 +- .../clientpositive/delete_all_partitioned.q.out |2 +- .../clientpositive/distinct_windowing.q.out |6 +- .../distinct_windowing_no_cbo.q.out | 10 +- .../groupby_grouping_window.q.out |2 +- .../results/clientpositive/input_part7.q.out|4 +- .../insert_values_non_partitioned.q.out |2 +- .../clientpositive/limit_pushdown2.q.out| 40 +- .../clientpositive/llap/acid_no_buckets.q.out | 16 +- .../llap/acid_vectorization_original.q.out |2 +- .../clientpositive/llap/bucketmapjoin7.q.out|2 +- .../results/clientpositive/llap/cbo_limit.q.out |8 +- .../clientpositive/llap/cbo_rp_limit.q.out |8 +- ...names_with_leading_and_trailing_spaces.q.out |2 +- .../llap/delete_all_partitioned.q.out |2 +- .../clientpositive/llap/explainuser_1.q.out | 88 +- .../llap/groupby_resolution.q.out |2 +- .../llap/insert_into_with_schema.q.out |4 +- .../llap/insert_values_non_partitioned.q.out|2 +- .../clientpositive/llap/limit_pushdown.q.out|8 +- .../clientpositive/llap/limit_pushdown3.q.out |8 +- .../results/clientpositive/llap/lineage2.q.out |2 +- .../results/clientpositive/llap/lineage3.q.out | 10 +- .../results/clientpositive/llap/llap_acid.q.out |6 +- .../clientpositive/llap/llap_acid_fast.q.out|6 +- .../clientpositive/llap/llap_smb_ptf.q.out |4 +- .../llap/offset_limit_ppd_optimizer.q.out |8 +- .../clientpositive/llap/orc_llap_counters.q.out | 82 +- .../llap/orc_llap_counters1.q.out | 10 +- .../clientpositive/llap/orc_ppd_basic.q.out | 100 +- .../llap/orc_ppd_schema_evol_3a.q.out | 112 +- .../clientpositive/llap/order_null.q.out|4 +- .../test/results/clientpositive/llap/ptf.q.out | 140 +-- .../clientpositive/llap/ptf_matchpath.q.out |6 +- .../clientpositive/llap/ptf_streaming.q.out | 94 +- .../clientpositive/llap/sharedworkext.q.out |4 +- .../clientpositive/llap/skewjoinopt15.q.out |4 +- .../clientpositive/llap/smb_mapjoin_15.q.out|8 +- .../clientpositive/llap/subquery_in.q.out |4 +- .../llap/subquery_in_having.q.out |2 +- .../clientpositive/llap/subquery_notin.q.out| 14 +- .../clientpositive/llap/subquery_scalar.q.out |4 +- .../llap/tez_dynpart_hashjoin_1.q.out |4 +- .../llap/tez_dynpart_hashjoin_2.q.out | 12 +- .../llap/tez_fixed_bucket_pruning.q.out | 24 +- .../llap/tez_vector_dynpart_hashjoin_1.q.out|4 +- .../llap/tez_vector_dynpart_hashjoin_2.q.out| 12 +- .../llap/update_all_partitioned.q.out |2 +- .../clientpositive/llap/update_tmp_table.q.out |2 +-
[19/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out index 4cfb1d9..8fadc45 100644 --- a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out @@ -1314,7 +1314,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: a +reduceColumnNullOrder: z reduceColumnSortOrder: + allNative: false usesVectorUDFAdaptor: false @@ -1521,7 +1521,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: a +reduceColumnNullOrder: z reduceColumnSortOrder: + allNative: false usesVectorUDFAdaptor: false @@ -1771,7 +1771,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: a +reduceColumnNullOrder: z reduceColumnSortOrder: + allNative: false usesVectorUDFAdaptor: false @@ -1794,7 +1794,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: a +reduceColumnNullOrder: z reduceColumnSortOrder: + allNative: false usesVectorUDFAdaptor: false @@ -2106,7 +2106,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: a +reduceColumnNullOrder: z reduceColumnSortOrder: + allNative: false usesVectorUDFAdaptor: false @@ -2313,7 +2313,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: a +reduceColumnNullOrder: z reduceColumnSortOrder: + allNative: false usesVectorUDFAdaptor: false @@ -2564,7 +2564,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: a +reduceColumnNullOrder: z reduceColumnSortOrder: + allNative: false usesVectorUDFAdaptor: false @@ -2587,7 +2587,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: a +reduceColumnNullOrder: z reduceColumnSortOrder: + allNative: false usesVectorUDFAdaptor: false http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out index 00be86c..9178652 100644 --- a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out @@ -97,7 +97,6 @@ POSTHOOK: query: select distinct si, si%4 from over10k_n2 order by si POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n2 A masked pattern was here -NULL NULL 2560 2571 2582 @@ -350,6 +349,7 @@ NULLNULL 5091 5102 5113 +NULL NULL PREHOOK: query: insert into over10k_orc_bucketed select * from
[12/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out index 748dea1..beaf86f 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out @@ -200,7 +200,7 @@ STAGE PLANS: Partition table definition input alias: ptf_1 name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int, _col7: double partition by: _col2 raw input shape: @@ -216,7 +216,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: aa +reduceColumnNullOrder: az reduceColumnSortOrder: ++ allNative: false usesVectorUDFAdaptor: false @@ -244,7 +244,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -515,7 +515,7 @@ STAGE PLANS: Partition table definition input alias: ptf_1 name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int partition by: _col2 raw input shape: @@ -547,7 +547,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -708,7 +708,7 @@ STAGE PLANS: Partition table definition input alias: ptf_1 name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int partition by: _col2 raw input shape: @@ -867,7 +867,7 @@ STAGE PLANS: Partition table definition input alias: abc name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int, _col7: double partition by: _col2 raw input shape: @@ -883,7 +883,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: aa +reduceColumnNullOrder: az reduceColumnSortOrder: ++ allNative: false usesVectorUDFAdaptor: false @@ -911,7 +911,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -1119,7 +1119,7 @@ STAGE PLANS: Partition table definition input alias: ptf_1 name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int partition by: _col2 raw input shape: @@ -1151,7 +1151,7 @@ STAGE PLANS: Windowing table definition
[08/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
--
diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
index 1a8ee5b..1a8583d 100644
--- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
@@ -2128,7 +2128,7 @@ Stage-0
Filter Operator [FIL_23] (rows=26 width=491)
predicate:first_value_window_0 is not null
PTF Operator [PTF_10] (rows=26 width=491)
-Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS
FIRST","partition by:":"_col2"}]
+Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS
LAST","partition by:":"_col2"}]
Select Operator [SEL_9] (rows=26 width=491)
Output:["_col1","_col2","_col5"]
<-Map 4 [PARTITION-LEVEL SORT]
@@ -2558,7 +2558,7 @@ Stage-0
Select Operator [SEL_4] (rows=20 width=64)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"]
PTF Operator [PTF_3] (rows=20 width=621)
-Function definitions:[{},{"name:":"windowingtablefunction","order
by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}]
+Function definitions:[{},{"name:":"windowingtablefunction","order
by:":"_col1 ASC NULLS LAST","partition by:":"_col0"}]
Select Operator [SEL_2] (rows=20 width=621)
Output:["_col0","_col1","_col2","_col3"]
<-Map 1 [PARTITION-LEVEL SORT]
@@ -2585,7 +2585,7 @@ Stage-0
Select Operator [SEL_4] (rows=25 width=179)
Output:["_col0","_col1","_col2"]
PTF Operator [PTF_3] (rows=25 width=443)
-Function definitions:[{},{"name:":"windowingtablefunction","order
by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}]
+Function definitions:[{},{"name:":"windowingtablefunction","order
by:":"_col1 ASC NULLS LAST","partition by:":"_col0"}]
Select Operator [SEL_2] (rows=25 width=443)
Output:["_col0","_col1"]
<-Map 1 [PARTITION-LEVEL SORT]
@@ -4057,14 +4057,14 @@ Stage-0
Select Operator [SEL_7] (rows=26 width=239)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
PTF Operator [PTF_6] (rows=26 width=499)
-Function definitions:[{},{"name:":"windowingtablefunction","order
by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}]
+Function definitions:[{},{"name:":"windowingtablefunction","order
by:":"_col1 ASC NULLS LAST","partition by:":"_col2"}]
Select Operator [SEL_5] (rows=26 width=499)
Output:["_col1","_col2","_col5","_col7"]
<-Reducer 2 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_4]
PartitionCols:_col2
PTF Operator [PTF_3] (rows=26 width=499)
- Function definitions:[{},{"Partition table
definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition
by:":"_col2"}}]
+ Function definitions:[{},{"Partition table
definition":{"name:":"noop","order by:":"_col1 ASC NULLS LAST","partition
by:":"_col2"}}]
Select Operator [SEL_2] (rows=26 width=499)
Output:["_col1","_col2","_col5","_col7"]
<-Map 1 [PARTITION-LEVEL SORT]
@@ -4102,14 +4102,14 @@ Stage-0
Select Operator [SEL_14] (rows=27 width=227)
Output:["_col0","_col1","_col2","_col3"]
PTF Operator [PTF_13] (rows=27 width=223)
-Function definitions:[{},{"name:":"windowingtablefunction","order
by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}]
+Function definitions:[{},{"name:":"windowingtablefunction","order
by:":"_col1 ASC NULLS LAST","partition by:":"_col2"}]
Select Operator [SEL_12] (rows=27 width=223)
Output:["_col1","_col2","_col5"]
<-Reducer 2 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_11]
PartitionCols:_col2
PTF Operator [PTF_10] (rows=27 width=223)
- Function definitions:[{},{"Partition table
definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition
by:":"_col2"}}]
+ Function definitions:[{},{"Partition table
definition":{"name:":"noop","order by:":"_col1 ASC NULLS LAST","partition
by:":"_col2"}}]
Select Operator [SEL_9] (rows=27 width=223)
Output:["_col1","_col2","_col5"]
<-Map 1 [PARTITION-LEVEL SORT]
@@ -4167,14 +4167,14 @@ Stage-0
Select
[09/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out -- diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out index f448a3e..7de4d38 100644 --- a/ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out @@ -291,8 +291,8 @@ from alltypesparquet where cbigint > 0 and cbigint < 1 order by s1, s2 l POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesparquet A masked pattern was here --985319NULL-0.01217879691754650 -9853192.0297994862577501E-4 -0.01217879691754650 +-985319NULL-0.01217879691754650 -63925 0.11256941728588189 -0.18771998435666797 0 NULLNULL 0 NULLNULL @@ -517,8 +517,8 @@ POSTHOOK: Input: default@alltypesparquet -273.0 6028764.868131869 1.0 6028764.868131869 -0.01098901098901099-0.004395604395604396 -257.0 6404096.533073931.0 6404096.53307393 -0.011673151750972763 -0.004669260700389105 -250.0 6583411.236 1.0 6583411.236 -0.012 -0.0048 --247.0 NULL1.0 NULL-0.012145748987854251 -0.004858299595141701 -247.0 -7546669.174089069 1.0 -7546669.174089069 -0.012145748987854251 -0.004858299595141701 +-247.0 NULL1.0 NULL-0.012145748987854251 -0.004858299595141701 -246.0 NULL1.0 NULL-0.012195121951219513 -0.004878048780487805 -237.0 NULL1.0 NULL-0.012658227848101266 -0.005063291139240506 -236.0 NULL1.0 NULL-0.012711864406779662 -0.005084745762711864 @@ -546,18 +546,18 @@ POSTHOOK: Input: default@alltypesparquet -132.0 NULL1.0 NULL-0.022727272727272728 -0.00909090909090909 -129.0 1.2758548906976745E71.0 1.2758548906976745E7 -0.023255813953488372 -0.009302325581395349 -128.0 NULL1.0 NULL-0.0234375 -0.009375 --126.0 NULL1.0 NULL-0.023809523809523808 -0.009523809523809523 -126.0 -1.4793867349206349E7 1.0 -1.4793867349206349E7 -0.023809523809523808 -0.009523809523809523 +-126.0 NULL1.0 NULL-0.023809523809523808 -0.009523809523809523 -116.0 NULL1.0 NULL-0.02586206896551724-0.010344827586206896 --113.0 NULL1.0 NULL-0.02654867256637168-0.010619469026548672 -113.0 -1.6495816690265486E7 1.0 -1.6495816690265486E7 -0.02654867256637168-0.010619469026548672 +-113.0 NULL1.0 NULL-0.02654867256637168-0.010619469026548672 -96.0 NULL1.0 NULL-0.03125-0.012499 -94.0 -1.9830077510638297E7 1.0 -1.9830077510638297E7 -0.031914893617021274 -0.01276595744680851 -93.0 NULL1.0 NULL-0.03225806451612903-0.012903225806451613 -77.0 2.4513789038961038E71.0 2.4513789038961038E7 -0.03896103896103896-0.015584415584415584 -69.0 2.735596747826087E7 1.0 2.735596747826087E7 -0.043478260869565216 -0.017391304347826087 --62.0 NULL1.0 NULL-0.04838709677419355-0.01935483870967742 -62.0 3.0444544451612905E71.0 3.0444544451612905E7 -0.04838709677419355-0.01935483870967742 +-62.0 NULL1.0 NULL-0.04838709677419355-0.01935483870967742 -60.0 NULL1.0 NULL-0.05 -0.02 -57.0 -3.27022330877193E7 1.0 -3.27022330877193E7 -0.05263157894736842-0.021052631578947368 -49.0 3.35888328367347E7 1.0 3.35888328367347E7 -0.061224489795918366 -0.024489795918367346 http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out -- diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out index b2c0a64..97194a2 100644 --- a/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out @@ -150,7 +150,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true -reduceColumnNullOrder: aa +reduceColumnNullOrder: zz reduceColumnSortOrder: ++ allNative: false usesVectorUDFAdaptor: false @@ -307,7 +307,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS
[14/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/llap/vector_windowing_expressions.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing_expressions.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing_expressions.q.out index 5ea866b..d470240 100644 --- a/ql/src/test/results/clientpositive/llap/vector_windowing_expressions.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_windowing_expressions.q.out @@ -134,7 +134,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col7 ASC NULLS FIRST +order by: _col7 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -311,7 +311,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: aa +reduceColumnNullOrder: az reduceColumnSortOrder: ++ allNative: false usesVectorUDFAdaptor: false @@ -339,7 +339,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col7 ASC NULLS FIRST +order by: _col7 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -520,7 +520,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col6 ASC NULLS FIRST, _col7 ASC NULLS FIRST, _col1 ASC NULLS FIRST, _col4 DESC NULLS LAST +order by: _col6 ASC NULLS LAST, _col7 ASC NULLS LAST, _col1 ASC NULLS LAST, _col4 DESC NULLS LAST partition by: _col0 raw input shape: window functions: @@ -743,7 +743,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col2 ASC NULLS FIRST, _col7 ASC NULLS FIRST +order by: _col2 ASC NULLS LAST, _col7 ASC NULLS LAST partition by: _col1 raw input shape: window functions: @@ -966,7 +966,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col1 ASC NULLS FIRST, _col7 ASC NULLS FIRST, _col5 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST, _col7 ASC NULLS LAST, _col5 ASC NULLS LAST partition by: _col3 raw input shape: window functions: @@ -1190,7 +1190,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col3 ASC NULLS FIRST +order by: _col3 ASC NULLS LAST partition by: _col4 raw input shape: window functions: @@ -1397,7 +1397,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -reduceColumnNullOrder: aa +reduceColumnNullOrder: za reduceColumnSortOrder: ++ allNative: false usesVectorUDFAdaptor: false @@ -1425,7 +1425,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col2 ASC NULLS FIRST +order by: _col2 ASC NULLS LAST partition by: _col2, _col4 raw input shape: window functions: @@ -1483,32 +1483,32 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part A masked pattern was here p_mfgr avg_window_0 -Manufacturer#1 1753.76 -Manufacturer#3 1410.39 -Manufacturer#4 1620.67 -Manufacturer#5 1018.1 -Manufacturer#5 1788.73 -Manufacturer#1 1173.15
[05/20] hive git commit: HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
http://git-wip-us.apache.org/repos/asf/hive/blob/499d01a3/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out -- diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index 0c48310..9eb1a2c 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -197,7 +197,7 @@ STAGE PLANS: Partition table definition input alias: ptf_1 name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int, _col7: double partition by: _col2 raw input shape: @@ -213,7 +213,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true -reduceColumnNullOrder: aa +reduceColumnNullOrder: az reduceColumnSortOrder: ++ allNative: false usesVectorUDFAdaptor: false @@ -241,7 +241,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -512,7 +512,7 @@ STAGE PLANS: Partition table definition input alias: ptf_1 name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int partition by: _col2 raw input shape: @@ -543,7 +543,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -701,7 +701,7 @@ STAGE PLANS: Partition table definition input alias: ptf_1 name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int partition by: _col2 raw input shape: @@ -857,7 +857,7 @@ STAGE PLANS: Partition table definition input alias: abc name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int, _col7: double partition by: _col2 raw input shape: @@ -873,7 +873,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true -reduceColumnNullOrder: aa +reduceColumnNullOrder: az reduceColumnSortOrder: ++ allNative: false usesVectorUDFAdaptor: false @@ -901,7 +901,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST partition by: _col2 raw input shape: window functions: @@ -1106,7 +1106,7 @@ STAGE PLANS: Partition table definition input alias: ptf_1 name: noop -order by: _col1 ASC NULLS FIRST +order by: _col1 ASC NULLS LAST output shape: _col1: string, _col2: string, _col5: int partition by: _col2 raw input shape: @@ -1137,7 +1137,7 @@ STAGE PLANS: Windowing table definition
[09/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java
--
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java
new file mode 100644
index 000..75f16cb
--- /dev/null
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.conf;
+
+import java.util.concurrent.TimeUnit;
+
+public class TimeValidator implements Validator {
+
+ private final TimeUnit unit;
+ private final Long min;
+ private final boolean minInclusive;
+
+ private final Long max;
+ private final boolean maxInclusive;
+
+ public TimeValidator(TimeUnit unit) {
+this(unit, null, false, null, false);
+ }
+
+ public TimeValidator(TimeUnit unit, Long min, boolean minInclusive, Long max,
+ boolean maxInclusive) {
+this.unit = unit;
+this.min = min;
+this.minInclusive = minInclusive;
+this.max = max;
+this.maxInclusive = maxInclusive;
+ }
+
+ @Override
+ public void validate(String value) {
+// First just check that this translates
+TimeUnit defaultUnit = unit;
+long time = MetastoreConf.convertTimeStr(value, defaultUnit, defaultUnit);
+if (min != null) {
+ if (minInclusive ? time < min : time <= min) {
+throw new IllegalArgumentException(value + " is smaller than minimum "
+ min +
+MetastoreConf.timeAbbreviationFor(defaultUnit));
+ }
+}
+
+if (max != null) {
+ if (maxInclusive ? time > max : time >= max) {
+throw new IllegalArgumentException(value + " is larger than maximum "
+ max +
+MetastoreConf.timeAbbreviationFor(defaultUnit));
+ }
+}
+ }
+
+ private String timeString(long time, TimeUnit timeUnit) {
+return time + " " + MetastoreConf.timeAbbreviationFor(timeUnit);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
--
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
new file mode 100644
index 000..91d790a
--- /dev/null
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.partition.spec;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static
[03/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
deleted file mode 100644
index 30ea7f8..000
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ /dev/null
@@ -1,1713 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.conf;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * A set of definitions of config values used by the Metastore. One of the
key aims of this
- * class is to provide backwards compatibility with existing Hive
configuration keys while
- * allowing the metastore to have its own, Hive independent keys. For this
reason access to the
- * underlying Configuration object should always be done via the static
methods provided here
- * rather than directly via {@link Configuration#get(String)} and
- * {@link Configuration#set(String, String)}. All the methods of this class
will handle checking
- * both the MetastoreConf key and the Hive key. The algorithm is, on reads,
to check first the
- * MetastoreConf key, then the Hive key, then return the default if neither
are set. On write
- * the Metastore key only is set.
- *
- * This class does not extend Configuration. Rather it provides static
methods for operating on
- * a Configuration object. This allows it to work on HiveConf objects, which
otherwise would not
- * be the case.
- */
-public class MetastoreConf {
-
- private static final Logger LOG =
LoggerFactory.getLogger(MetastoreConf.class);
- private static final Pattern TIME_UNIT_SUFFIX =
Pattern.compile("([0-9]+)([a-zA-Z]+)");
-
- @VisibleForTesting
- static final String DEFAULT_STORAGE_SCHEMA_READER_CLASS =
- "org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader";
- @VisibleForTesting
- static final String HIVE_ALTER_HANDLE_CLASS =
- "org.apache.hadoop.hive.metastore.HiveAlterHandler";
- @VisibleForTesting
- static final String MATERIALZIATIONS_REBUILD_LOCK_CLEANER_TASK_CLASS =
-
"org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask";
- @VisibleForTesting
- static final String METASTORE_TASK_THREAD_CLASS =
- "org.apache.hadoop.hive.metastore.MetastoreTaskThread";
- @VisibleForTesting
- static final String RUNTIME_STATS_CLEANER_TASK_CLASS =
- "org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask";
- @VisibleForTesting
- static final String EVENT_CLEANER_TASK_CLASS =
- "org.apache.hadoop.hive.metastore.events.EventCleanerTask";
- @VisibleForTesting
- static final String METASTORE_DELEGATION_MANAGER_CLASS =
-
"org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager";
- @VisibleForTesting
- static final String ACID_COMPACTION_HISTORY_SERVICE_CLASS =
- "org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService";
- @VisibleForTesting
- static final String ACID_HOUSE_KEEPER_SERVICE_CLASS =
- "org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService";
- @VisibleForTesting
- static final String ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS =
- "org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService";
- @VisibleForTesting
- static final String
[04/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
deleted file mode 100644
index ddcda4c..000
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-/**
- * Define a set of APIs that may vary in different environments
- */
-public interface MetaStoreFS {
-
- /**
- * delete a directory
- *
- * @param f
- * @param ifPurge
- * @param recursive
- * @return true on success
- * @throws MetaException
- */
- public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
- boolean ifPurge, Configuration conf) throws MetaException;
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetadataStore.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetadataStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetadataStore.java
deleted file mode 100644
index 26e2c49..000
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetadataStore.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-public interface MetadataStore {
- /**
- * @param fileIds file ID list.
- * @param result The ref parameter, used to return the serialized file
metadata.
- */
- void getFileMetadata(List fileIds, ByteBuffer[] result) throws
IOException;
-
- /**
- * @param fileIds file ID list.
- * @param metadataBuffers Serialized file metadata, one per file ID.
- * @param addedCols The column names for additional columns created by
file-format-specific
- * metadata handler, to be stored in the cache.
- * @param addedVals The values for addedCols; one value per file ID per
added column.
- */
- void storeFileMetadata(List fileIds, List metadataBuffers,
- ByteBuffer[] addedCols, ByteBuffer[][] addedVals) throws IOException,
InterruptedException;
-
- /**
- * @param fileId The file ID.
- * @param metadata Serialized file metadata.
- * @param addedCols The column names for additional columns created by
file-format-specific
- * metadata handler, to be stored in the cache.
- * @param addedVals The values for addedCols; one value per added column.
- */
- void storeFileMetadata(long fileId, ByteBuffer metadata, ByteBuffer[]
addedCols,
- ByteBuffer[] addedVals) throws IOException, InterruptedException;
-
-}
[13/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
--
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
new file mode 100644
index 000..df6d56b
--- /dev/null
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -0,0 +1,3637 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.security.auth.login.LoginException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.ValidTxnList;
+import org.apache.hadoop.hive.common.ValidWriteIdList;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.hooks.URIResolverHook;
+import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.utils.ObjectPair;
+import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.thrift.TApplicationException;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.transport.TFramedTransport;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
+/**
+ * Hive Metastore Client.
+ * The public implementation of IMetaStoreClient. Methods not inherited from
IMetaStoreClient
+ * are not public and can change. Hence this is marked as unstable.
+ * For users who require retry mechanism when the connection between metastore
and client is
+ * broken, RetryingMetaStoreClient class should be used.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
+ /**
+ * Capabilities of
[07/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
deleted file mode 100644
index 35be3c4..000
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-
-
-/**
- * A class that defines the constant strings used by the statistics
implementation.
- */
-
-public class StatsSetupConst {
-
- protected static final Logger LOG =
LoggerFactory.getLogger(StatsSetupConst.class.getName());
-
- public enum StatDB {
-fs {
- @Override
- public String getPublisher(Configuration conf) {
-return "org.apache.hadoop.hive.ql.stats.fs.FSStatsPublisher";
- }
-
- @Override
- public String getAggregator(Configuration conf) {
-return "org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator";
- }
-},
-custom {
- @Override
- public String getPublisher(Configuration conf) {
-return MetastoreConf.getVar(conf, ConfVars.STATS_DEFAULT_PUBLISHER); }
- @Override
- public String getAggregator(Configuration conf) {
-return MetastoreConf.getVar(conf, ConfVars.STATS_DEFAULT_AGGREGATOR);
}
-};
-public abstract String getPublisher(Configuration conf);
-public abstract String getAggregator(Configuration conf);
- }
-
- // statistics stored in metastore
- /**
- * The name of the statistic Num Files to be published or gathered.
- */
- public static final String NUM_FILES = "numFiles";
-
- /**
- * The name of the statistic Num Partitions to be published or gathered.
- */
- public static final String NUM_PARTITIONS = "numPartitions";
-
- /**
- * The name of the statistic Total Size to be published or gathered.
- */
- public static final String TOTAL_SIZE = "totalSize";
-
- /**
- * The name of the statistic Row Count to be published or gathered.
- */
- public static final String ROW_COUNT = "numRows";
-
- public static final String RUN_TIME_ROW_COUNT = "runTimeNumRows";
-
- /**
- * The name of the statistic Raw Data Size to be published or gathered.
- */
- public static final String RAW_DATA_SIZE = "rawDataSize";
-
- /**
- * The name of the statistic for Number of Erasure Coded Files - to be
published or gathered.
- */
- public static final String NUM_ERASURE_CODED_FILES = "numFilesErasureCoded";
-
- /**
- * Temp dir for writing stats from tasks.
- */
- public static final String STATS_TMP_LOC = "hive.stats.tmp.loc";
-
- public static final String STATS_FILE_PREFIX = "tmpstats-";
- /**
- * List of all
[10/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
--
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
new file mode 100644
index 000..30ea7f8
--- /dev/null
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -0,0 +1,1713 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.conf;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.utils.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A set of definitions of config values used by the Metastore. One of the
key aims of this
+ * class is to provide backwards compatibility with existing Hive
configuration keys while
+ * allowing the metastore to have its own, Hive independent keys. For this
reason access to the
+ * underlying Configuration object should always be done via the static
methods provided here
+ * rather than directly via {@link Configuration#get(String)} and
+ * {@link Configuration#set(String, String)}. All the methods of this class
will handle checking
+ * both the MetastoreConf key and the Hive key. The algorithm is, on reads,
to check first the
+ * MetastoreConf key, then the Hive key, then return the default if neither
are set. On write
+ * the Metastore key only is set.
+ *
+ * This class does not extend Configuration. Rather it provides static
methods for operating on
+ * a Configuration object. This allows it to work on HiveConf objects, which
otherwise would not
+ * be the case.
+ */
+public class MetastoreConf {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(MetastoreConf.class);
+ private static final Pattern TIME_UNIT_SUFFIX =
Pattern.compile("([0-9]+)([a-zA-Z]+)");
+
+ @VisibleForTesting
+ static final String DEFAULT_STORAGE_SCHEMA_READER_CLASS =
+ "org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader";
+ @VisibleForTesting
+ static final String HIVE_ALTER_HANDLE_CLASS =
+ "org.apache.hadoop.hive.metastore.HiveAlterHandler";
+ @VisibleForTesting
+ static final String MATERIALZIATIONS_REBUILD_LOCK_CLEANER_TASK_CLASS =
+
"org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask";
+ @VisibleForTesting
+ static final String METASTORE_TASK_THREAD_CLASS =
+ "org.apache.hadoop.hive.metastore.MetastoreTaskThread";
+ @VisibleForTesting
+ static final String RUNTIME_STATS_CLEANER_TASK_CLASS =
+ "org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask";
+ @VisibleForTesting
+ static final String EVENT_CLEANER_TASK_CLASS =
+ "org.apache.hadoop.hive.metastore.events.EventCleanerTask";
+ @VisibleForTesting
+ static final String METASTORE_DELEGATION_MANAGER_CLASS =
+
"org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager";
+ @VisibleForTesting
+ static final String ACID_COMPACTION_HISTORY_SERVICE_CLASS =
+ "org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService";
+ @VisibleForTesting
+ static final String ACID_HOUSE_KEEPER_SERVICE_CLASS =
+ "org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService";
+ @VisibleForTesting
+ static final String ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS =
+ "org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService";
+ @VisibleForTesting
+ static final String
[08/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
--
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
new file mode 100644
index 000..155ecb1
--- /dev/null
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
@@ -0,0 +1,597 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.utils;
+
+import org.apache.curator.shaded.com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class FileUtils {
+ private static final PathFilter SNAPSHOT_DIR_PATH_FILTER = new PathFilter() {
+@Override
+public boolean accept(Path p) {
+ return ".snapshot".equalsIgnoreCase(p.getName());
+}
+ };
+ private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class);
+
+ public static final PathFilter HIDDEN_FILES_PATH_FILTER = new PathFilter() {
+@Override
+public boolean accept(Path p) {
+ String name = p.getName();
+ return !name.startsWith("_") && !name.startsWith(".");
+}
+ };
+ /**
+ * Filter that filters out hidden files
+ */
+ private static final PathFilter hiddenFileFilter = new PathFilter() {
+@Override
+public boolean accept(Path p) {
+ String name = p.getName();
+ return !name.startsWith("_") && !name.startsWith(".");
+}
+ };
+
+ /**
+ * Move a particular file or directory to the trash.
+ * @param fs FileSystem to use
+ * @param f path of file or directory to move to trash.
+ * @param conf configuration object
+ * @return true if move successful
+ * @throws IOException
+ */
+ public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf,
boolean purge)
+ throws IOException {
+LOG.debug("deleting " + f);
+boolean result;
+try {
+ if(purge) {
+LOG.debug("purge is set to true. Not moving to Trash " + f);
+ } else {
+result = Trash.moveToAppropriateTrash(fs, f, conf);
+if (result) {
+ LOG.trace("Moved to trash: " + f);
+ return true;
+}
+ }
+} catch (IOException ioe) {
+ // for whatever failure reason including that trash has lower encryption
zone
+ // retry with force delete
+ LOG.warn(ioe.getMessage() + "; Force to delete it.");
+}
+
+result = fs.delete(f, true);
+if (!result) {
+ LOG.error("Failed to delete " + f);
+}
+return result;
+ }
+
+ /**
+ * Copies files between filesystems.
+ */
+ public static boolean copy(FileSystem srcFS, Path src,
+ FileSystem dstFS, Path dst,
+ boolean deleteSource,
+ boolean overwrite,
+ Configuration conf) throws IOException {
+boolean copied = false;
+boolean triedDistcp = false;
+
+/* Run distcp if source file/dir is too big */
+if (srcFS.getUri().getScheme().equals("hdfs")) {
+ ContentSummary srcContentSummary = srcFS.getContentSummary(src);
+ if (srcContentSummary.getFileCount() >
+MetastoreConf.getLongVar(conf,
[14/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
HIVE-20483 Really move metastore common classes into metastore-common
(Alexander Kolbasov via Alan Gates).
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/35f86c74
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/35f86c74
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/35f86c74
Branch: refs/heads/master
Commit: 35f86c749cefc2a9972a991deed78a1c3719093d
Parents: 265
Author: Alan Gates
Authored: Thu Sep 13 12:38:43 2018 -0700
Committer: Alan Gates
Committed: Thu Sep 13 12:38:43 2018 -0700
--
beeline/pom.xml |5 +
hcatalog/hcatalog-pig-adapter/pom.xml |7 +
hcatalog/pom.xml|7 +
metastore/pom.xml |5 -
ql/pom.xml |5 +
.../hadoop/hive/common/StatsSetupConst.java | 336 ++
.../hadoop/hive/metastore/ColumnType.java | 301 ++
.../hive/metastore/FileMetadataHandler.java | 109 +
.../hive/metastore/HiveMetaStoreClient.java | 3637 +
.../hadoop/hive/metastore/IMetaStoreClient.java | 3761 ++
.../hadoop/hive/metastore/MetaStoreFS.java | 43 +
.../hadoop/hive/metastore/MetadataStore.java| 52 +
.../hive/metastore/MetastoreTaskThread.java | 38 +
.../metastore/PartitionExpressionProxy.java | 73 +
.../hive/metastore/ReplChangeManager.java | 502 +++
.../hive/metastore/RetryingMetaStoreClient.java | 341 ++
.../hadoop/hive/metastore/TableIterable.java| 115 +
.../apache/hadoop/hive/metastore/Warehouse.java | 759
.../hive/metastore/conf/MetastoreConf.java | 1713
.../hive/metastore/conf/TimeValidator.java | 67 +
.../spec/CompositePartitionSpecProxy.java | 258 ++
.../spec/PartitionListComposingSpecProxy.java | 209 +
.../partition/spec/PartitionSpecProxy.java | 220 +
.../spec/PartitionSpecWithSharedSDProxy.java| 192 +
.../security/DelegationTokenIdentifier.java | 52 +
.../security/DelegationTokenSecretManager.java | 134 +
.../security/DelegationTokenSelector.java | 33 +
.../security/HadoopThriftAuthBridge.java| 700
.../security/HadoopThriftAuthBridge23.java | 114 +
.../hadoop/hive/metastore/utils/FileUtils.java | 597 +++
.../hadoop/hive/metastore/utils/HdfsUtils.java | 395 ++
.../hive/metastore/utils/MetaStoreUtils.java| 908 +
.../hive/metastore/utils/SecurityUtils.java | 270 ++
.../hadoop/hive/common/StatsSetupConst.java | 336 --
.../hadoop/hive/metastore/ColumnType.java | 301 --
.../hive/metastore/FileMetadataHandler.java | 109 -
.../hive/metastore/HiveMetaStoreClient.java | 3637 -
.../hadoop/hive/metastore/IMetaStoreClient.java | 3761 --
.../hadoop/hive/metastore/MetaStoreFS.java | 43 -
.../hadoop/hive/metastore/MetadataStore.java| 52 -
.../hive/metastore/MetastoreTaskThread.java | 38 -
.../metastore/PartitionExpressionProxy.java | 73 -
.../hive/metastore/ReplChangeManager.java | 502 ---
.../hive/metastore/RetryingMetaStoreClient.java | 341 --
.../hadoop/hive/metastore/TableIterable.java| 115 -
.../apache/hadoop/hive/metastore/Warehouse.java | 759
.../hive/metastore/conf/MetastoreConf.java | 1713
.../hive/metastore/conf/TimeValidator.java | 67 -
.../spec/CompositePartitionSpecProxy.java | 258 --
.../spec/PartitionListComposingSpecProxy.java | 209 -
.../partition/spec/PartitionSpecProxy.java | 220 -
.../spec/PartitionSpecWithSharedSDProxy.java| 192 -
.../security/DelegationTokenIdentifier.java | 52 -
.../security/DelegationTokenSecretManager.java | 134 -
.../security/DelegationTokenSelector.java | 33 -
.../security/HadoopThriftAuthBridge.java| 700
.../security/HadoopThriftAuthBridge23.java | 114 -
.../hadoop/hive/metastore/utils/FileUtils.java | 597 ---
.../hadoop/hive/metastore/utils/HdfsUtils.java | 395 --
.../hive/metastore/utils/MetaStoreUtils.java| 908 -
.../hive/metastore/utils/SecurityUtils.java | 270 --
streaming/pom.xml |6 +
62 files changed, 15959 insertions(+), 15934 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/beeline/pom.xml
--
diff --git a/beeline/pom.xml b/beeline/pom.xml
index 4567d5e..19ec53e 100644
--- a/beeline/pom.xml
+++ b/beeline/pom.xml
@@ -55,6 +55,11 @@
hive-jdbc
${project.version}
+
+ org.apache.hive
+ hive-standalone-metastore-server
+ ${project.version}
+
commons-cli
[05/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java -- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java deleted file mode 100644 index 54e7eda..000 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ /dev/null @@ -1,3761 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidWriteIdList; -import org.apache.hadoop.hive.common.classification.RetrySemantics; -import org.apache.hadoop.hive.metastore.annotation.NoReconnect; -import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; -import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; -import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.CmRecycleRequest; -import org.apache.hadoop.hive.metastore.api.CmRecycleResponse; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.CommitTxnRequest; -import org.apache.hadoop.hive.metastore.api.CompactionResponse; -import org.apache.hadoop.hive.metastore.api.CompactionType; -import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; -import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; -import org.apache.hadoop.hive.metastore.api.DataOperationType; -import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.FindSchemasByColsResp; -import org.apache.hadoop.hive.metastore.api.FindSchemasByColsRqst; -import org.apache.hadoop.hive.metastore.api.FireEventRequest; -import org.apache.hadoop.hive.metastore.api.FireEventResponse; -import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest; -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse; -import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; -import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleRequest; -import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleResponse; -import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest; -import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalResponse; -import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse; -import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; -import org.apache.hadoop.hive.metastore.api.HiveObjectRef; -import org.apache.hadoop.hive.metastore.api.ISchema; -import org.apache.hadoop.hive.metastore.api.InvalidInputException; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; -import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; -import org.apache.hadoop.hive.metastore.api.LockRequest; -import org.apache.hadoop.hive.metastore.api.LockResponse; -import org.apache.hadoop.hive.metastore.api.Materialization; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.MetadataPpdResult; -import
[02/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java
deleted file mode 100644
index 75f16cb..000
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.conf;
-
-import java.util.concurrent.TimeUnit;
-
-public class TimeValidator implements Validator {
-
- private final TimeUnit unit;
- private final Long min;
- private final boolean minInclusive;
-
- private final Long max;
- private final boolean maxInclusive;
-
- public TimeValidator(TimeUnit unit) {
-this(unit, null, false, null, false);
- }
-
- public TimeValidator(TimeUnit unit, Long min, boolean minInclusive, Long max,
- boolean maxInclusive) {
-this.unit = unit;
-this.min = min;
-this.minInclusive = minInclusive;
-this.max = max;
-this.maxInclusive = maxInclusive;
- }
-
- @Override
- public void validate(String value) {
-// First just check that this translates
-TimeUnit defaultUnit = unit;
-long time = MetastoreConf.convertTimeStr(value, defaultUnit, defaultUnit);
-if (min != null) {
- if (minInclusive ? time < min : time <= min) {
-throw new IllegalArgumentException(value + " is smaller than minimum "
+ min +
-MetastoreConf.timeAbbreviationFor(defaultUnit));
- }
-}
-
-if (max != null) {
- if (maxInclusive ? time > max : time >= max) {
-throw new IllegalArgumentException(value + " is larger than maximum "
+ max +
-MetastoreConf.timeAbbreviationFor(defaultUnit));
- }
-}
- }
-
- private String timeString(long time, TimeUnit timeUnit) {
-return time + " " + MetastoreConf.timeAbbreviationFor(timeUnit);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
deleted file mode 100644
index 91d790a..000
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore.partition.spec;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static
[01/14] hive git commit: HIVE-20483 Really move metastore common classes into metastore-common (Alexander Kolbasov via Alan Gates).
Repository: hive
Updated Branches:
refs/heads/master 26543 -> 35f86c749
http://git-wip-us.apache.org/repos/asf/hive/blob/35f86c74/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
deleted file mode 100644
index 155ecb1..000
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
+++ /dev/null
@@ -1,597 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.utils;
-
-import org.apache.curator.shaded.com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.Trash;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-public class FileUtils {
- private static final PathFilter SNAPSHOT_DIR_PATH_FILTER = new PathFilter() {
-@Override
-public boolean accept(Path p) {
- return ".snapshot".equalsIgnoreCase(p.getName());
-}
- };
- private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class);
-
- public static final PathFilter HIDDEN_FILES_PATH_FILTER = new PathFilter() {
-@Override
-public boolean accept(Path p) {
- String name = p.getName();
- return !name.startsWith("_") && !name.startsWith(".");
-}
- };
- /**
- * Filter that filters out hidden files
- */
- private static final PathFilter hiddenFileFilter = new PathFilter() {
-@Override
-public boolean accept(Path p) {
- String name = p.getName();
- return !name.startsWith("_") && !name.startsWith(".");
-}
- };
-
- /**
- * Move a particular file or directory to the trash.
- * @param fs FileSystem to use
- * @param f path of file or directory to move to trash.
- * @param conf configuration object
- * @return true if move successful
- * @throws IOException
- */
- public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf,
boolean purge)
- throws IOException {
-LOG.debug("deleting " + f);
-boolean result;
-try {
- if(purge) {
-LOG.debug("purge is set to true. Not moving to Trash " + f);
- } else {
-result = Trash.moveToAppropriateTrash(fs, f, conf);
-if (result) {
- LOG.trace("Moved to trash: " + f);
- return true;
-}
- }
-} catch (IOException ioe) {
- // for whatever failure reason including that trash has lower encryption
zone
- // retry with force delete
- LOG.warn(ioe.getMessage() + "; Force to delete it.");
-}
-
-result = fs.delete(f, true);
-if (!result) {
- LOG.error("Failed to delete " + f);
-}
-return result;
- }
-
- /**
- * Copies files between filesystems.
- */
- public static boolean copy(FileSystem srcFS, Path src,
- FileSystem dstFS, Path dst,
- boolean deleteSource,
- boolean overwrite,
- Configuration conf) throws IOException {
-boolean copied = false;
-boolean triedDistcp = false;
-
-/* Run distcp if source file/dir is too big */
-if (srcFS.getUri().getScheme().equals("hdfs")) {
- ContentSummary srcContentSummary = srcFS.getContentSummary(src);
- if
hive git commit: HIVE-20541: REPL DUMP on external table with add partition event throws NoSuchElementException (Sankar Hariappan, reviewed by Anishek Agarwal)
Repository: hive
Updated Branches:
refs/heads/branch-3 f1d8c3f61 -> b4e552df1
HIVE-20541: REPL DUMP on external table with add partition event throws
NoSuchElementException (Sankar Hariappan, reviewed by Anishek Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4e552df
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4e552df
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4e552df
Branch: refs/heads/branch-3
Commit: b4e552df109f3fc15bf9769b43986b455dc62cb1
Parents: f1d8c3f
Author: Sankar Hariappan
Authored: Thu Sep 13 14:49:47 2018 +0530
Committer: Sankar Hariappan
Committed: Thu Sep 13 14:49:47 2018 +0530
--
...TestReplicationScenariosAcrossInstances.java | 21
.../repl/dump/events/AddPartitionHandler.java | 21
2 files changed, 34 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/b4e552df/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 6c45641..35ad982 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1400,4 +1400,25 @@ public class TestReplicationScenariosAcrossInstances {
.run("select id from t4")
.verifyResult(null); // Returns null as create table event doesn't
list files
}
+
+ @Test
+ public void testDumpExternalTableWithAddPartitionEvent() throws Throwable {
+WarehouseInstance.Tuple tuple = primary.dump("repl dump " + primaryDbName);
+
+replica.load(replicatedDbName, tuple.dumpLocation);
+
+tuple = primary.run("use " + primaryDbName)
+.run("create external table t1 (place string) partitioned by
(country string)")
+.run("alter table t1 add partition(country='india')")
+.run("alter table t1 add partition(country='us')")
+.dump("repl dump " + primaryDbName + " from " +
tuple.lastReplicationId
++ " with ('hive.repl.include.external.tables'='true')");
+
+replica.load(replicatedDbName, tuple.dumpLocation)
+.run("use " + replicatedDbName)
+.run("show tables like 't1'")
+.verifyResult("t1")
+.run("show partitions t1")
+.verifyResults(new String[] { "country=india", "country=us" });
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b4e552df/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
index cf15905..973a65b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
@@ -85,14 +85,19 @@ class AddPartitionHandler extends AbstractEventHandler {
withinContext.hiveConf);
Iterator partitionFilesIter =
apm.getPartitionFilesIter().iterator();
-for (Partition qlPtn : qlPtns) {
- Iterable files = partitionFilesIter.next().getFiles();
- if (files != null) {
-// encoded filename/checksum of files, write into _files
-try (BufferedWriter fileListWriter = writer(withinContext, qlPtn)) {
- for (String file : files) {
-fileListWriter.write(file);
-fileListWriter.newLine();
+
+// We expect one to one mapping between partitions and file iterators. For
external table, this
+// list would be empty. So, it is enough to check hasNext outside the loop.
+if (partitionFilesIter.hasNext()) {
+ for (Partition qlPtn : qlPtns) {
+Iterable files = partitionFilesIter.next().getFiles();
+if (files != null) {
+ // encoded filename/checksum of files, write into _files
+ try (BufferedWriter fileListWriter = writer(withinContext, qlPtn)) {
+for (String file : files) {
+ fileListWriter.write(file);
+ fileListWriter.newLine();
+}
}
}
}
hive git commit: HIVE-20541: REPL DUMP on external table with add partition event throws NoSuchElementException (Sankar Hariappan, reviewed by Anishek Agarwal)
Repository: hive
Updated Branches:
refs/heads/master b7723e498 -> 26543
HIVE-20541: REPL DUMP on external table with add partition event throws
NoSuchElementException (Sankar Hariappan, reviewed by Anishek Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2654
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2654
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2654
Branch: refs/heads/master
Commit: 265433d00900f437cd2d5e7893d0870754df
Parents: b7723e4
Author: Sankar Hariappan
Authored: Thu Sep 13 14:48:46 2018 +0530
Committer: Sankar Hariappan
Committed: Thu Sep 13 14:48:46 2018 +0530
--
...TestReplicationScenariosAcrossInstances.java | 21
.../repl/dump/events/AddPartitionHandler.java | 21
2 files changed, 34 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/2654/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 6c45641..35ad982 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1400,4 +1400,25 @@ public class TestReplicationScenariosAcrossInstances {
.run("select id from t4")
.verifyResult(null); // Returns null as create table event doesn't
list files
}
+
+ @Test
+ public void testDumpExternalTableWithAddPartitionEvent() throws Throwable {
+WarehouseInstance.Tuple tuple = primary.dump("repl dump " + primaryDbName);
+
+replica.load(replicatedDbName, tuple.dumpLocation);
+
+tuple = primary.run("use " + primaryDbName)
+.run("create external table t1 (place string) partitioned by
(country string)")
+.run("alter table t1 add partition(country='india')")
+.run("alter table t1 add partition(country='us')")
+.dump("repl dump " + primaryDbName + " from " +
tuple.lastReplicationId
++ " with ('hive.repl.include.external.tables'='true')");
+
+replica.load(replicatedDbName, tuple.dumpLocation)
+.run("use " + replicatedDbName)
+.run("show tables like 't1'")
+.verifyResult("t1")
+.run("show partitions t1")
+.verifyResults(new String[] { "country=india", "country=us" });
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2654/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
index cf15905..973a65b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
@@ -85,14 +85,19 @@ class AddPartitionHandler extends AbstractEventHandler {
withinContext.hiveConf);
Iterator partitionFilesIter =
apm.getPartitionFilesIter().iterator();
-for (Partition qlPtn : qlPtns) {
- Iterable files = partitionFilesIter.next().getFiles();
- if (files != null) {
-// encoded filename/checksum of files, write into _files
-try (BufferedWriter fileListWriter = writer(withinContext, qlPtn)) {
- for (String file : files) {
-fileListWriter.write(file);
-fileListWriter.newLine();
+
+// We expect one to one mapping between partitions and file iterators. For
external table, this
+// list would be empty. So, it is enough to check hasNext outside the loop.
+if (partitionFilesIter.hasNext()) {
+ for (Partition qlPtn : qlPtns) {
+Iterable files = partitionFilesIter.next().getFiles();
+if (files != null) {
+ // encoded filename/checksum of files, write into _files
+ try (BufferedWriter fileListWriter = writer(withinContext, qlPtn)) {
+for (String file : files) {
+ fileListWriter.write(file);
+ fileListWriter.newLine();
+}
}
}
}
hive git commit: HIVE-20503: Use datastructure aware estimations during mapjoin selection (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master a3b7a2452 -> b7723e498
HIVE-20503: Use datastructure aware estimations during mapjoin selection
(Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b7723e49
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b7723e49
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b7723e49
Branch: refs/heads/master
Commit: b7723e4987ee5efc8990c92866e76055754b7bf1
Parents: a3b7a24
Author: Zoltan Haindrich
Authored: Thu Sep 13 09:21:42 2018 +0200
Committer: Zoltan Haindrich
Committed: Thu Sep 13 09:21:42 2018 +0200
--
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 63 ++--
.../tez/TestVectorMapJoinFastHashTable.java | 4 +-
2 files changed, 61 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/b7723e49/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index cd952a2..c733cb1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -68,6 +68,9 @@ import org.apache.hadoop.hive.ql.plan.OpTraits;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
+import
org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,6 +91,8 @@ public class ConvertJoinMapJoin implements NodeProcessor {
private static final Logger LOG =
LoggerFactory.getLogger(ConvertJoinMapJoin.class.getName());
public float hashTableLoadFactor;
private long maxJoinMemory;
+ private HashMapDataStructureType hashMapDataStructure;
+ private boolean fastHashTableAvailable;
@Override
/*
@@ -102,6 +107,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
OptimizeTezProcContext context = (OptimizeTezProcContext) procCtx;
hashTableLoadFactor =
context.conf.getFloatVar(ConfVars.HIVEHASHTABLELOADFACTOR);
+fastHashTableAvailable =
context.conf.getBoolVar(ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_FAST_HASHTABLE_ENABLED);
JoinOperator joinOp = (JoinOperator) nd;
// adjust noconditional task size threshold for LLAP
@@ -116,6 +122,9 @@ public class ConvertJoinMapJoin implements NodeProcessor {
LOG.info("maxJoinMemory: {}", maxJoinMemory);
+hashMapDataStructure = HashMapDataStructureType.of(joinOp.getConf());
+
+
TezBucketJoinProcCtx tezBucketJoinProcCtx = new
TezBucketJoinProcCtx(context.conf);
boolean hiveConvertJoin =
context.conf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN) &
!context.parseContext.getDisableMapJoin();
@@ -193,6 +202,32 @@ public class ConvertJoinMapJoin implements NodeProcessor {
return null;
}
+ private enum HashMapDataStructureType {
+COMPOSITE_KEYED, LONG_KEYED;
+
+public static HashMapDataStructureType of(JoinDesc conf) {
+ ExprNodeDesc[][] keys = conf.getJoinKeys();
+ if (keys != null && keys[0].length == 1) {
+TypeInfo typeInfo = keys[0][0].getTypeInfo();
+if (typeInfo instanceof PrimitiveTypeInfo) {
+ PrimitiveTypeInfo pti = ((PrimitiveTypeInfo) typeInfo);
+ PrimitiveCategory pCat = pti.getPrimitiveCategory();
+ switch (pCat) {
+ case BOOLEAN:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+return HashMapDataStructureType.LONG_KEYED;
+ default:
+break;
+ }
+}
+ }
+ return HashMapDataStructureType.COMPOSITE_KEYED;
+}
+ }
+
private boolean selectJoinForLlap(OptimizeTezProcContext context,
JoinOperator joinOp,
TezBucketJoinProcCtx tezBucketJoinProcCtx,
LlapClusterStateForCompile llapInfo,
@@ -239,6 +274,11 @@ public class ConvertJoinMapJoin implements NodeProcessor {
LOG.info("Cost of Bucket Map Join : numNodes = " + numNodes + " total
small table size = "
+ totalSize + " networkCostMJ = " + networkCostMJ);
+if (totalSize <= maxJoinMemory) {
+ // mapjoin is applicable; don't try the below algos..
+ return false;
+}
+
if (networkCostDPHJ <
