BIGTOP-885. TestHiveSmokeBulk fails on Hive 0.9
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8d32a92d Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8d32a92d Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8d32a92d Branch: refs/heads/master Commit: 8d32a92d04a4aeca41b56dc67f5bf4832eec392b Parents: 7b2f77d Author: Konstantin Boudnik <[email protected]> Authored: Wed Apr 24 13:29:54 2013 -0700 Committer: Konstantin Boudnik <[email protected]> Committed: Wed Apr 24 13:29:54 2013 -0700 ---------------------------------------------------------------------- .../resources/scripts/ql/authorization_2/filter | 2 - .../main/resources/scripts/ql/authorization_2/in | 123 -- .../main/resources/scripts/ql/authorization_2/out | 59 - .../main/resources/scripts/ql/auto_join20/filter | 4 +- .../src/main/resources/scripts/ql/auto_join20/out | 632 +++----- .../src/main/resources/scripts/ql/basic/filter | 3 +- .../hive/src/main/resources/scripts/ql/basic/out | 18 +- .../scripts/ql/bucketizedhiveinputformat/filter | 4 +- .../scripts/ql/bucketizedhiveinputformat/out | 24 +- .../resources/scripts/ql/bucketmapjoin5/filter | 4 +- .../main/resources/scripts/ql/bucketmapjoin5/out | 482 ++++-- .../resources/scripts/ql/drop_multi_partitions/out | 16 +- .../ql/groupby_map_ppr_multi_distinct/filter | 4 +- .../scripts/ql/groupby_map_ppr_multi_distinct/out | 164 +- .../resources/scripts/ql/index_creation/filter | 4 +- .../main/resources/scripts/ql/index_creation/out | 46 +- .../src/main/resources/scripts/ql/join19/filter | 1 + .../hive/src/main/resources/scripts/ql/join19/out | 232 ++-- .../main/resources/scripts/ql/join_filters/filter | 3 +- .../src/main/resources/scripts/ql/join_filters/out | 24 +- .../resources/scripts/ql/load_dyn_part14/filter | 6 +- .../main/resources/scripts/ql/load_dyn_part14/out | 160 +- .../scripts/ql/merge_dynamic_partition/filter | 4 +- .../scripts/ql/merge_dynamic_partition/out | 226 ++-- .../main/resources/scripts/ql/multi_insert/filter | 5 +- .../src/main/resources/scripts/ql/multi_insert/out | 1252 ++++++++------- .../resources/scripts/ql/rcfile_columnar/filter | 3 +- .../main/resources/scripts/ql/rcfile_columnar/out | 20 +- .../src/main/resources/scripts/ql/stats8/filter | 6 - .../hive/src/main/resources/scripts/ql/stats8/in | 47 - .../hive/src/main/resources/scripts/ql/stats8/out | 224 --- .../src/main/resources/scripts/ql/union3/filter | 4 +- .../hive/src/main/resources/scripts/ql/union3/out | 128 +- .../main/resources/scripts/ql/uniquejoin/filter | 3 +- .../src/main/resources/scripts/ql/uniquejoin/out | 20 +- 35 files changed, 1722 insertions(+), 2235 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/filter ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/filter b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/filter deleted file mode 100644 index 015b318..0000000 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/filter +++ /dev/null @@ -1,2 +0,0 @@ -sed -e 's#^grantTime.[0-9]*#grantTime\tJUSTNOW#' \ - -e 's#^grantor.*$#grantor\t\tBORG#' http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/in ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/in b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/in deleted file mode 100644 index 015fe80..0000000 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/in +++ /dev/null @@ -1,123 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License") you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -create table authorization_part (key int, value string) partitioned by (ds string); - -ALTER TABLE authorization_part SET TBLPROPERTIES ("PARTITION_LEVEL_PRIVILEGE"="TRUE"); -set hive.security.authorization.enabled=true; - --- column grant to user -grant Create on table authorization_part to user hive_test_user; -grant Update on table authorization_part to user hive_test_user; -grant Drop on table authorization_part to user hive_test_user; -grant select on table src to user hive_test_user; - -show grant user hive_test_user on table authorization_part; - -alter table authorization_part add partition (ds='2010'); -show grant user hive_test_user on table authorization_part partition (ds='2010'); - -grant select(key) on table authorization_part to user hive_test_user; -alter table authorization_part drop partition (ds='2010'); -insert overwrite table authorization_part partition (ds='2010') select key, value from src; -show grant user hive_test_user on table authorization_part(key) partition (ds='2010'); -show grant user hive_test_user on table authorization_part(key); -select key from authorization_part where ds='2010' order by key limit 20; - -revoke select(key) on table authorization_part from user hive_test_user; -show grant user hive_test_user on table authorization_part(key); -show grant user hive_test_user on table authorization_part(key) partition (ds='2010'); - -select key from authorization_part where ds='2010' order by key limit 20; - -revoke select(key) on table authorization_part partition (ds='2010') from user hive_test_user; -show grant user hive_test_user on table authorization_part(key) partition (ds='2010'); - -alter table authorization_part drop partition (ds='2010'); - --- table grant to user -show grant user hive_test_user on table authorization_part; - -alter table authorization_part add partition (ds='2010'); -show grant user hive_test_user on table authorization_part partition (ds='2010'); - -grant select on table authorization_part to user hive_test_user; -alter table authorization_part drop partition (ds='2010'); -insert overwrite table authorization_part partition (ds='2010') select key, value from src; -show grant user hive_test_user on table authorization_part partition (ds='2010'); -show grant user hive_test_user on table authorization_part; -select key from authorization_part where ds='2010' order by key limit 20; - -revoke select on table authorization_part from user hive_test_user; -show grant user hive_test_user on table authorization_part; -show grant user hive_test_user on table authorization_part partition (ds='2010'); - -select key from authorization_part where ds='2010' order by key limit 20; - -revoke select on table authorization_part partition (ds='2010') from user hive_test_user; -show grant user hive_test_user on table authorization_part partition (ds='2010'); - -alter table authorization_part drop partition (ds='2010'); - --- column grant to group - -show grant group hive_test_group1 on table authorization_part; - -alter table authorization_part add partition (ds='2010'); -show grant group hive_test_group1 on table authorization_part partition (ds='2010'); - -grant select(key) on table authorization_part to group hive_test_group1; -alter table authorization_part drop partition (ds='2010'); -insert overwrite table authorization_part partition (ds='2010') select key, value from src; -show grant group hive_test_group1 on table authorization_part(key) partition (ds='2010'); -show grant group hive_test_group1 on table authorization_part(key); -select key from authorization_part where ds='2010' order by key limit 20; - -revoke select(key) on table authorization_part from group hive_test_group1; -show grant group hive_test_group1 on table authorization_part(key); -show grant group hive_test_group1 on table authorization_part(key) partition (ds='2010'); - -select key from authorization_part where ds='2010' order by key limit 20; - -revoke select(key) on table authorization_part partition (ds='2010') from group hive_test_group1; -show grant group hive_test_group1 on table authorization_part(key) partition (ds='2010'); - -alter table authorization_part drop partition (ds='2010'); - --- table grant to group -show grant group hive_test_group1 on table authorization_part; - -alter table authorization_part add partition (ds='2010'); -show grant group hive_test_group1 on table authorization_part partition (ds='2010'); - -grant select on table authorization_part to group hive_test_group1; -alter table authorization_part drop partition (ds='2010'); -insert overwrite table authorization_part partition (ds='2010') select key, value from src; -show grant group hive_test_group1 on table authorization_part partition (ds='2010'); -show grant group hive_test_group1 on table authorization_part; -select key from authorization_part where ds='2010' order by key limit 20; - -revoke select on table authorization_part from group hive_test_group1; -show grant group hive_test_group1 on table authorization_part; -show grant group hive_test_group1 on table authorization_part partition (ds='2010'); - -select key from authorization_part where ds='2010' order by key limit 20; - -revoke select on table authorization_part partition (ds='2010') from group hive_test_group1; -show grant group hive_test_group1 on table authorization_part partition (ds='2010'); - - -revoke select on table src from user hive_test_user; -set hive.security.authorization.enabled=false; -drop table authorization_part; http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/out ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/out b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/out deleted file mode 100644 index acae64c..0000000 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/authorization_2/out +++ /dev/null @@ -1,59 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License") you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. -create table authorization_part (key int, value string) partitioned by (ds string) - - -ALTER TABLE authorization_part SET TBLPROPERTIES ("PARTITION_LEVEL_PRIVILEGE"="TRUE") -set hive.security.authorization.enabled=true - - --- column grant to user -grant Create on table authorization_part to user hive_test_user - -grant Update on table authorization_part to user hive_test_user - -grant Drop on table authorization_part to user hive_test_user - -grant select on table src to user hive_test_user - - -show grant user hive_test_user on table authorization_part - -database default -table authorization_part -principalName hive_test_user -principalType USER -privilege Create -grantTime 1301676873 -grantor hudson - -database default -table authorization_part -principalName hive_test_user -principalType USER -privilege Update -grantTime 1301676874 -grantor hudson - -database default -table authorization_part -principalName hive_test_user -principalType USER -privilege Drop -grantTime 1301676874 -grantor hudson - - -alter table authorization_part add partition (ds='2010') http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/filter ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/filter b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/filter index e67bc0d..9cdeb19 100644 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/filter +++ b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/filter @@ -1 +1,3 @@ -sed -re 's#hdfs://.*/-(ext|mr)-1000#hdfs://HADOOP/-\1-1000#' +sed -re 's#hdfs://.*/-(ext|mr)-1000#hdfs://HADOOP/-\1-1000#' \ + -e 's#file:/.*/-(ext|mr)-1000#file:/HADOOP/-\1-1000#' \ + -e '/.*jobconf.xml:an attempt to override final parameter: mapreduce.job.end-notification.*; Ignoring\./ d' http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/out ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/out b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/out index b58f461..092f64d 100644 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/out +++ b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/auto_join20/out @@ -1,17 +1,3 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License") you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. set hive.auto.convert.join = true @@ -26,27 +12,25 @@ ABSTRACT SYNTAX TREE: (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) src1) (TOK_TABREF (TOK_TABNAME src) src2) (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)) (< (. (TOK_TABLE_OR_COL src1) key) 10))) (TOK_TABREF (TOK_TABNAME src) src3) (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src3) key)) (< (. (TOK_TABLE_OR_COL src3) key) 20)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) v1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) v2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) key) k3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) value) v3)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v2)) (TOK_TABSO RTCOLNAMEASC (TOK_TABLE_OR_COL k3)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v3))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL a) v1) (. (TOK_TABLE_OR_COL a) k2) (. (TOK_TABLE_OR_COL a) v2) (. (TOK_TABLE_OR_COL a) k3) (. (TOK_TABLE_OR_COL a) v3))))))) STAGE DEPENDENCIES: - Stage-8 is a root stage , consists of Stage-9, Stage-10, Stage-1 - Stage-9 has a backup stage: Stage-1 - Stage-6 depends on stages: Stage-9 - Stage-2 depends on stages: Stage-1, Stage-6, Stage-7 + Stage-7 is a root stage , consists of Stage-8, Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-6 depends on stages: Stage-8 + Stage-2 depends on stages: Stage-1, Stage-6 Stage-3 depends on stages: Stage-2 - Stage-10 has a backup stage: Stage-1 - Stage-7 depends on stages: Stage-10 Stage-1 Stage-0 is a root stage STAGE PLANS: - Stage: Stage-8 + Stage: Stage-7 Conditional Operator - Stage: Stage-9 + Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: a:src1 Fetch Operator limit: -1 - a:src3 + a:src2 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: @@ -55,52 +39,52 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10) + expr: (key < 10.0) type: boolean - Filter Operator - predicate: - expr: (key < 10) - type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 1 - a:src3 + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 + 2 {(key < 20.0)} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 + a:src2 TableScan - alias: src3 - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 1 + alias: src2 + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 + 2 {(key < 20.0)} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 Stage: Stage-6 Map Reduce Alias -> Map Operator Tree: - a:src2 + a:src3 TableScan - alias: src2 + alias: src3 Map Join Operator condition map: Inner Join 0 to 1 @@ -112,14 +96,14 @@ STAGE PLANS: filter predicates: 0 1 - 2 {(key < 20)} + 2 {(key < 20.0)} handleSkewJoin: false keys: 0 [Column[key]] 1 [Column[key]] 2 [Column[key]] outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Position of Big Table: 1 + Position of Big Table: 2 Select Operator expressions: expr: _col0 @@ -147,7 +131,7 @@ STAGE PLANS: Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - hdfs://monster01.sf.cloudera.com:17020/tmp/hive-hudson/hive_2011-04-01_10-21-00_772_4864094195694055998/-mr-10002 + hdfs://mgrover-bigtop-centos-1.ent.cloudera.com:17020/tmp/hive-root/hive_2013-04-23_11-35-58_309_5301659382942807999/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -210,7 +194,7 @@ STAGE PLANS: Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - hdfs://monster01.sf.cloudera.com:17020/tmp/hive-hudson/hive_2011-04-01_10-21-00_772_4864094195694055998/-mr-10003 + hdfs://mgrover-bigtop-centos-1.ent.cloudera.com:17020/tmp/hive-root/hive_2013-04-23_11-35-58_309_5301659382942807999/-mr-10003 Reduce Output Operator sort order: tag: -1 @@ -236,155 +220,51 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-10 - Map Reduce Local Work - Alias -> Map Local Tables: - a:src1 - Fetch Operator - limit: -1 - a:src2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: a:src1 TableScan alias: src1 Filter Operator predicate: - expr: (key < 10) + expr: (key < 10.0) type: boolean - Filter Operator - predicate: - expr: (key < 10) - type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 - a:src2 - TableScan - alias: src2 - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 - - Stage: Stage-7 - Map Reduce - Alias -> Map Operator Tree: - a:src3 - TableScan - alias: src3 - Map Join Operator - condition map: - Inner Join 0 to 1 - Right Outer Join0 to 2 - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Position of Big Table: 2 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 + Reduce Output Operator + key expressions: + expr: key type: string - expr: _col5 + sort order: + + Map-reduce partition columns: + expr: key type: string - expr: _col8 + tag: 0 + value expressions: + expr: key type: string - expr: _col9 + expr: value type: string - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - Local Work: - Map Reduce Local Work - - Stage: Stage-1 - Map Reduce - Alias -> Map Operator Tree: - a:src1 + a:src2 TableScan - alias: src1 + alias: src2 Filter Operator predicate: - expr: (key < 10) + expr: (key < 10.0) type: boolean - Filter Operator - predicate: - expr: (key < 10) - type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: value - type: string - a:src2 - TableScan - alias: src2 - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string a:src3 TableScan alias: src3 @@ -414,7 +294,7 @@ STAGE PLANS: filter predicates: 0 1 - 2 {(VALUE._col0 < 20)} + 2 {(VALUE._col0 < 20.0)} handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 Select Operator @@ -466,27 +346,25 @@ ABSTRACT SYNTAX TREE: (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) src1) (TOK_TABREF (TOK_TABNAME src) src2) (AND (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)) (< (. (TOK_TABLE_OR_COL src1) key) 10)) (< (. (TOK_TABLE_OR_COL src2) key) 15))) (TOK_TABREF (TOK_TABNAME src) src3) (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src3) key)) (< (. (TOK_TABLE_OR_COL src3) key) 20)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) v1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) v2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) key) k3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) value) v3)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k2)) (TOK_TABSOR TCOLNAMEASC (TOK_TABLE_OR_COL v2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k3)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v3))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL a) v1) (. (TOK_TABLE_OR_COL a) k2) (. (TOK_TABLE_OR_COL a) v2) (. (TOK_TABLE_OR_COL a) k3) (. (TOK_TABLE_OR_COL a) v3))))))) STAGE DEPENDENCIES: - Stage-8 is a root stage , consists of Stage-9, Stage-10, Stage-1 - Stage-9 has a backup stage: Stage-1 - Stage-6 depends on stages: Stage-9 - Stage-2 depends on stages: Stage-1, Stage-6, Stage-7 + Stage-7 is a root stage , consists of Stage-8, Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-6 depends on stages: Stage-8 + Stage-2 depends on stages: Stage-1, Stage-6 Stage-3 depends on stages: Stage-2 - Stage-10 has a backup stage: Stage-1 - Stage-7 depends on stages: Stage-10 Stage-1 Stage-0 is a root stage STAGE PLANS: - Stage: Stage-8 + Stage: Stage-7 Conditional Operator - Stage: Stage-9 + Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: a:src1 Fetch Operator limit: -1 - a:src3 + a:src2 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: @@ -495,31 +373,56 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10) + expr: ((key < 10.0) and (key < 15.0)) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 + 2 {(key < 20.0)} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 + a:src2 + TableScan + alias: src2 + Filter Operator + predicate: + expr: ((key < 15.0) and (key < 10.0)) type: boolean - Filter Operator - predicate: - expr: (key < 10) - type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 1 + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 + 2 {(key < 20.0)} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 + + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: a:src3 TableScan alias: src3 - HashTable Sink Operator + Map Join Operator + condition map: + Inner Join 0 to 1 + Right Outer Join0 to 2 condition expressions: 0 {key} {value} 1 {key} {value} @@ -527,75 +430,42 @@ STAGE PLANS: filter predicates: 0 1 - 2 {(key < 20)} + 2 {(key < 20.0)} handleSkewJoin: false keys: 0 [Column[key]] 1 [Column[key]] 2 [Column[key]] - Position of Big Table: 1 - - Stage: Stage-6 - Map Reduce - Alias -> Map Operator Tree: - a:src2 - TableScan - alias: src2 - Filter Operator - predicate: - expr: (key < 15) - type: boolean - Filter Operator - predicate: - expr: (key < 15) - type: boolean - Map Join Operator - condition map: - Inner Join 0 to 1 - Right Outer Join0 to 2 - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Position of Big Table: 1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string - expr: _col8 - type: string - expr: _col9 - type: string - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Position of Big Table: 2 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col4 + type: string + expr: _col5 + type: string + expr: _col8 + type: string + expr: _col9 + type: string + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Local Work: Map Reduce Local Work Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - hdfs://monster01.sf.cloudera.com:17020/tmp/hive-hudson/hive_2011-04-01_10-21-48_735_5280603062175628505/-mr-10002 + hdfs://mgrover-bigtop-centos-1.ent.cloudera.com:17020/tmp/hive-root/hive_2013-04-23_11-37-20_822_7360885266220174309/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -658,7 +528,7 @@ STAGE PLANS: Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - hdfs://monster01.sf.cloudera.com:17020/tmp/hive-hudson/hive_2011-04-01_10-21-48_735_5280603062175628505/-mr-10003 + hdfs://mgrover-bigtop-centos-1.ent.cloudera.com:17020/tmp/hive-root/hive_2013-04-23_11-37-20_822_7360885266220174309/-mr-10003 Reduce Output Operator sort order: tag: -1 @@ -684,171 +554,51 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-10 - Map Reduce Local Work - Alias -> Map Local Tables: - a:src1 - Fetch Operator - limit: -1 - a:src2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: a:src1 TableScan alias: src1 Filter Operator predicate: - expr: (key < 10) + expr: ((key < 10.0) and (key < 15.0)) type: boolean - Filter Operator - predicate: - expr: (key < 10) - type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 - a:src2 - TableScan - alias: src2 - Filter Operator - predicate: - expr: (key < 15) - type: boolean - Filter Operator - predicate: - expr: (key < 15) - type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 - - Stage: Stage-7 - Map Reduce - Alias -> Map Operator Tree: - a:src3 - TableScan - alias: src3 - Map Join Operator - condition map: - Inner Join 0 to 1 - Right Outer Join0 to 2 - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Position of Big Table: 2 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 + Reduce Output Operator + key expressions: + expr: key type: string - expr: _col4 - type: string - expr: _col5 + sort order: + + Map-reduce partition columns: + expr: key type: string - expr: _col8 + tag: 0 + value expressions: + expr: key type: string - expr: _col9 + expr: value type: string - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - Local Work: - Map Reduce Local Work - - Stage: Stage-1 - Map Reduce - Alias -> Map Operator Tree: - a:src1 - TableScan - alias: src1 - Filter Operator - predicate: - expr: (key < 10) - type: boolean - Filter Operator - predicate: - expr: (key < 10) - type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: value - type: string a:src2 TableScan alias: src2 Filter Operator predicate: - expr: (key < 15) + expr: ((key < 15.0) and (key < 10.0)) type: boolean - Filter Operator - predicate: - expr: (key < 15) - type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string a:src3 TableScan alias: src3 @@ -878,7 +628,7 @@ STAGE PLANS: filter predicates: 0 1 - 2 {(VALUE._col0 < 20)} + 2 {(VALUE._col0 < 20.0)} handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 Select Operator http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter index cb844d1..0bba73c 100644 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter +++ b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/filter @@ -1,3 +1,4 @@ sed -e 's#hdfs://[^/]*/#hdfs://HADOOP/#' \ -e 's#Copying file:.*u.data#Copying file: u.data#' \ - -e '/^Deleted.*u_data$/d' + -e '/^Deleted.*u_data$/d' \ + -e '/.*jobconf.xml:an attempt to override final parameter: mapreduce.job.end-notification.*; Ignoring\./ d' http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/out ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/out b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/out index 5e94525..87b086b 100644 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/out +++ b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/basic/out @@ -1,17 +1,3 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License") you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. DROP TABLE u_data @@ -27,8 +13,8 @@ STORED AS TEXTFILE LOAD DATA LOCAL INPATH 'seed_data_files/ml-data/u.data' OVERWRITE INTO TABLE u_data -Copying file: file:/var/lib/hudson/workspace/Nightly-smoke-testing-monster-clone/examples/hive/target/seed_data_files/ml-data/u.data -Deleted hdfs://monster01.sf.cloudera.com:17020/user/hive/warehouse/u_data +Copying file: file:/root/bigtop/bigtop-tests/test-execution/smokes/hive/target/seed_data_files/ml-data/u.data +Deleted /user/hive/warehouse/u_data INSERT OVERWRITE DIRECTORY '/tmp/count' http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/filter ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/filter b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/filter index 26c4a38..b65fd71 100644 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/filter +++ b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/filter @@ -1,3 +1,5 @@ sed -re 's#hdfs://.*/-(ext|mr)-1000#hdfs://HADOOP/-\1-1000#' \ -e 's#Copying file:.*/kv..txt#Copying file:kvX.txt#' \ - -e '/^Deleted hdfs:/d' + -e '/^Deleted hdfs:/d' \ + -e 's#file:/.*/-(ext|mr)-1000#file:/HADOOP/-\1-1000#' \ + -e '/.*jobconf.xml:an attempt to override final parameter: mapreduce.job.end-notification.*; Ignoring\./ d' http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/out ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/out b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/out index 945016d..00ab302 100644 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/out +++ b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketizedhiveinputformat/out @@ -1,17 +1,3 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License") you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat set mapred.min.split.size = 64 @@ -20,7 +6,7 @@ CREATE TABLE T1(name STRING) STORED AS TEXTFILE LOAD DATA LOCAL INPATH 'seed_data_files/kv1.txt' INTO TABLE T1 -Copying file: file:/var/lib/hudson/workspace/Nightly-smoke-testing-monster/examples/hive/target/seed_data_files/kv1.txt +Copying file: file:/root/bigtop/bigtop-tests/test-execution/smokes/hive/target/seed_data_files/kv1.txt CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE @@ -161,7 +147,7 @@ STAGE PLANS: Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - hdfs://monster01.sf.cloudera.com:17020/tmp/hive-hudson/hive_2011-04-01_12-29-52_242_1273761163922951124/-mr-10003 + hdfs://mgrover-bigtop-centos-1.ent.cloudera.com:17020/tmp/hive-root/hive_2013-04-23_11-41-40_405_3858259332039900449/-mr-10003 Reduce Output Operator sort order: tag: -1 @@ -203,7 +189,7 @@ SELECT tmp1.name as name FROM ( JOIN (SELECT 'MMM' AS n FROM T1) tmp2 JOIN (SELECT 'MMM' AS n FROM T1) tmp3 ON tmp1.n = tmp2.n AND tmp1.n = tmp3.n) ttt LIMIT 5000000 -Deleted hdfs://monster01.sf.cloudera.com:17020/user/hive/warehouse/t2 +Deleted /user/hive/warehouse/t2 EXPLAIN SELECT COUNT(1) FROM T2 @@ -266,10 +252,10 @@ SELECT COUNT(1) FROM T2 CREATE TABLE T3(name STRING) STORED AS TEXTFILE LOAD DATA LOCAL INPATH 'seed_data_files/kv1.txt' INTO TABLE T3 -Copying file: file:/var/lib/hudson/workspace/Nightly-smoke-testing-monster/examples/hive/target/seed_data_files/kv1.txt +Copying file: file:/root/bigtop/bigtop-tests/test-execution/smokes/hive/target/seed_data_files/kv1.txt LOAD DATA LOCAL INPATH 'seed_data_files/kv2.txt' INTO TABLE T3 -Copying file: file:/var/lib/hudson/workspace/Nightly-smoke-testing-monster/examples/hive/target/seed_data_files/kv2.txt +Copying file: file:/root/bigtop/bigtop-tests/test-execution/smokes/hive/target/seed_data_files/kv2.txt EXPLAIN SELECT COUNT(1) FROM T3 http://git-wip-us.apache.org/repos/asf/bigtop/blob/8d32a92d/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketmapjoin5/filter ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketmapjoin5/filter b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketmapjoin5/filter index 69b7bb7..2384b8a 100644 --- a/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketmapjoin5/filter +++ b/bigtop-tests/test-artifacts/hive/src/main/resources/scripts/ql/bucketmapjoin5/filter @@ -1,4 +1,6 @@ sed -re 's#hdfs://[^/]*/#hdfs://HADOOP/#g' \ -e 's#Copying file:.*/srcbucket2#Copying file:srcbucket2#' \ -e 's#hdfs://.*/-(ext|mr)-1000#hdfs://HADOOP/-\1-1000#' \ - -e 's#transient_lastDdlTime [0-9]*#transient_lastDdlTime JUSTNOW#' + -e 's#transient_lastDdlTime [0-9]*#transient_lastDdlTime JUSTNOW#' \ + -e 's#file:/.*/-(ext|mr)-1000#file:/HADOOP/-\1-1000#' \ + -e '/.*jobconf.xml:an attempt to override final parameter: mapreduce.job.end-notification.*; Ignoring\./ d'
