svn commit: r1645825 - in /hive/branches/branch-0.14: data/files/ itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/test/org/apache/hadoop/hive/ql/optimizer/p
Author: vikram Date: Tue Dec 16 02:42:06 2014 New Revision: 1645825 URL: http://svn.apache.org/r1645825 Log: HIVE-8866: Vectorization on partitioned table throws ArrayIndexOutOfBoundsException when partitions are not of same #of columns (Matt McCline via Vikram Dixit K) Added: hive/branches/branch-0.14/data/files/inventory hive/branches/branch-0.14/ql/src/test/queries/clientpositive/vector_partition_diff_num_cols.q hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vector_partition_diff_num_cols.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/vector_partition_diff_num_cols.q.out Modified: hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java Added: hive/branches/branch-0.14/data/files/inventory URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/data/files/inventory?rev=1645825view=auto == --- hive/branches/branch-0.14/data/files/inventory (added) +++ hive/branches/branch-0.14/data/files/inventory Tue Dec 16 02:42:06 2014 @@ -0,0 +1,100 @@ +2450815|1|1|211| +2450815|2|1|235| +2450815|4|1|859| +2450815|7|1|704| +2450815|8|1|891| +2450815|10|1|834| +2450815|13|1|591| +2450815|14|1|579| +2450815|16|1|622| +2450815|19|1|745| +2450815|20|1|405| +2450815|22|1|| +2450815|25|1|676| +2450815|26|1|865| +2450815|28|1|24| +2450815|31|1|170| +2450815|32|1|797| +2450815|34|1|332| +2450815|37|1|618| +2450815|38|1|909| +2450815|40|1|738| +2450815|43|1|699| +2450815|44|1|975| +2450815|46|1|551| +2450815|49|1|857| +2450815|50|1|894| +2450815|52|1|279| +2450815|55|1|206| +2450815|56|1|746| +2450815|58|1|| +2450815|61|1|154| +2450815|62|1|888| +2450815|64|1|773| +2450815|67|1|107| +2450815|68|1|801| +2450815|70|1|749| +2450815|73|1|419| +2450815|74|1|541| +2450815|76|1|359| +2450815|79|1|578| +2450815|80|1|42| +2450815|82|1|83| +2450815|85|1|192| +2450815|86|1|567| +2450815|88|1|710| +2450815|91|1|282| +2450815|92|1|631| +2450815|94|1|587| +2450815|97|1|645| +2450815|98|1|667| +2450815|100|1|26| +2450815|103|1|224| +2450815|104|1|901| +2450815|106|1|887| +2450815|109|1|455| +2450815|110|1|784| +2450815|112|1|742| +2450815|115|1|978| +2450815|116|1|207| +2450815|118|1|462| +2450815|121|1|406| +2450815|122|1|559| +2450815|124|1|999| +2450815|127|1|870| +2450815|128|1|982| +2450815|130|1|700| +2450815|133|1|26| +2450815|134|1|356| +2450815|136|1|143| +2450815|139|1|838| +2450815|140|1|846| +2450815|142|1|657| +2450815|145|1|181| +2450815|146|1|730| +2450815|148|1|49| +2450815|151|1|652| +2450815|152|1|861| +2450815|154|1|329| +2450815|157|1|286| +2450815|158|1|| +2450815|160|1|62| +2450815|163|1|894| +2450815|164|1|463| +2450815|166|1|458| +2450815|169|1|696| +2450815|170|1|450| +2450815|172|1|842| +2450815|175|1|79| +2450815|176|1|260| +2450815|178|1|894| +2450815|181|1|4| +2450815|182|1|281| +2450815|184|1|797| +2450815|187|1|981| +2450815|188|1|58| +2450815|190|1|834| +2450815|193|1|54| +2450815|194|1|856| +2450815|196|1|803| +2450815|199|1|296| Modified: hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties?rev=1645825r1=1645824r2=1645825view=diff == --- hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties (original) +++ hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties Tue Dec 16 02:42:06 2014 @@ -190,6 +190,7 @@ minitez.query.files.shared=alter_merge_2 vector_mapjoin_reduce.q,\ vector_non_string_partition.q,\ vector_orderby_5.q,\ + vector_partition_diff_num_cols.q,\ vector_partitioned_date_time.q,\ vector_reduce_groupby_decimal.q,\ vector_string_concat.q,\ Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1645825r1=1645824r2=1645825view=diff == --- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original) +++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Tue Dec 16 02:42:06 2014 @@ -27,6 +27,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.Stack; import java.util.TreeMap; @@ -35,6 +36,7 @@ import java.util.regex.Pattern; import
svn commit: r1645826 - in /hive/trunk: data/files/ itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src
Author: vikram Date: Tue Dec 16 02:42:11 2014 New Revision: 1645826 URL: http://svn.apache.org/r1645826 Log: HIVE-8866: Vectorization on partitioned table throws ArrayIndexOutOfBoundsException when partitions are not of same #of columns (Matt McCline via Vikram Dixit K) Added: hive/trunk/data/files/inventory hive/trunk/ql/src/test/queries/clientpositive/vector_partition_diff_num_cols.q hive/trunk/ql/src/test/results/clientpositive/tez/vector_partition_diff_num_cols.q.out hive/trunk/ql/src/test/results/clientpositive/vector_partition_diff_num_cols.q.out Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java Added: hive/trunk/data/files/inventory URL: http://svn.apache.org/viewvc/hive/trunk/data/files/inventory?rev=1645826view=auto == --- hive/trunk/data/files/inventory (added) +++ hive/trunk/data/files/inventory Tue Dec 16 02:42:11 2014 @@ -0,0 +1,100 @@ +2450815|1|1|211| +2450815|2|1|235| +2450815|4|1|859| +2450815|7|1|704| +2450815|8|1|891| +2450815|10|1|834| +2450815|13|1|591| +2450815|14|1|579| +2450815|16|1|622| +2450815|19|1|745| +2450815|20|1|405| +2450815|22|1|| +2450815|25|1|676| +2450815|26|1|865| +2450815|28|1|24| +2450815|31|1|170| +2450815|32|1|797| +2450815|34|1|332| +2450815|37|1|618| +2450815|38|1|909| +2450815|40|1|738| +2450815|43|1|699| +2450815|44|1|975| +2450815|46|1|551| +2450815|49|1|857| +2450815|50|1|894| +2450815|52|1|279| +2450815|55|1|206| +2450815|56|1|746| +2450815|58|1|| +2450815|61|1|154| +2450815|62|1|888| +2450815|64|1|773| +2450815|67|1|107| +2450815|68|1|801| +2450815|70|1|749| +2450815|73|1|419| +2450815|74|1|541| +2450815|76|1|359| +2450815|79|1|578| +2450815|80|1|42| +2450815|82|1|83| +2450815|85|1|192| +2450815|86|1|567| +2450815|88|1|710| +2450815|91|1|282| +2450815|92|1|631| +2450815|94|1|587| +2450815|97|1|645| +2450815|98|1|667| +2450815|100|1|26| +2450815|103|1|224| +2450815|104|1|901| +2450815|106|1|887| +2450815|109|1|455| +2450815|110|1|784| +2450815|112|1|742| +2450815|115|1|978| +2450815|116|1|207| +2450815|118|1|462| +2450815|121|1|406| +2450815|122|1|559| +2450815|124|1|999| +2450815|127|1|870| +2450815|128|1|982| +2450815|130|1|700| +2450815|133|1|26| +2450815|134|1|356| +2450815|136|1|143| +2450815|139|1|838| +2450815|140|1|846| +2450815|142|1|657| +2450815|145|1|181| +2450815|146|1|730| +2450815|148|1|49| +2450815|151|1|652| +2450815|152|1|861| +2450815|154|1|329| +2450815|157|1|286| +2450815|158|1|| +2450815|160|1|62| +2450815|163|1|894| +2450815|164|1|463| +2450815|166|1|458| +2450815|169|1|696| +2450815|170|1|450| +2450815|172|1|842| +2450815|175|1|79| +2450815|176|1|260| +2450815|178|1|894| +2450815|181|1|4| +2450815|182|1|281| +2450815|184|1|797| +2450815|187|1|981| +2450815|188|1|58| +2450815|190|1|834| +2450815|193|1|54| +2450815|194|1|856| +2450815|196|1|803| +2450815|199|1|296| Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1645826r1=1645825r2=1645826view=diff == --- hive/trunk/itests/src/test/resources/testconfiguration.properties (original) +++ hive/trunk/itests/src/test/resources/testconfiguration.properties Tue Dec 16 02:42:11 2014 @@ -205,6 +205,7 @@ minitez.query.files.shared=alter_merge_2 vector_mapjoin_reduce.q,\ vector_non_string_partition.q,\ vector_orderby_5.q,\ + vector_partition_diff_num_cols.q,\ vector_partitioned_date_time.q,\ vector_reduce_groupby_decimal.q,\ vector_string_concat.q,\ Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1645826r1=1645825r2=1645826view=diff == --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Tue Dec 16 02:42:11 2014 @@ -27,6 +27,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.Stack; import java.util.TreeMap; @@ -35,6 +36,7 @@ import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.*; import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; import
svn commit: r1646050 - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/tez/
Author: vikram
Date: Tue Dec 16 19:22:35 2014
New Revision: 1646050
URL: http://svn.apache.org/r1646050
Log:
HIVE-9055: Tez: union all followed by group by followed by another union all
gives error (Vikram Dixit K, reviewed by Prasanth J)
Added:
hive/trunk/ql/src/test/queries/clientpositive/tez_multi_union.q
hive/trunk/ql/src/test/results/clientpositive/tez/tez_multi_union.q.out
Modified:
hive/trunk/itests/src/test/resources/testconfiguration.properties
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1646050r1=1646049r2=1646050view=diff
==
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Tue Dec
16 19:22:35 2014
@@ -268,6 +268,7 @@ minitez.query.files.shared=alter_merge_2
auto_sortmerge_join_8.q,\
auto_sortmerge_join_9.q
+
minitez.query.files=bucket_map_join_tez1.q,\
bucket_map_join_tez2.q,\
dynamic_partition_pruning.q,\
@@ -288,7 +289,8 @@ minitez.query.files=bucket_map_join_tez1
tez_union_group_by.q,\
tez_smb_main.q,\
tez_smb_1.q,\
- vectorized_dynamic_partition_pruning.q
+ vectorized_dynamic_partition_pruning.q,\
+ tez_multi_union.q
beeline.positive.exclude=add_part_exist.q,\
alter1.q,\
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java?rev=1646050r1=1646049r2=1646050view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java Tue
Dec 16 19:22:35 2014
@@ -111,7 +111,7 @@ public class GenTezWork implements NodeP
// will result into a vertex with multiple FS or RS operators.
if (context.childToWorkMap.containsKey(operator)) {
// if we've seen both root and child, we can bail.
-
+
// clear out the mapjoin set. we don't need it anymore.
context.currentMapJoinOperators.clear();
@@ -349,17 +349,20 @@ public class GenTezWork implements NodeP
} else if (followingWork instanceof UnionWork) {
// this can only be possible if there is merge work followed by the
union
UnionWork unionWork = (UnionWork) followingWork;
- int index = getMergeIndex(tezWork, unionWork, rs);
- // guaranteed to be instance of MergeJoinWork if index is valid
- BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
- if (baseWork instanceof MergeJoinWork) {
-MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
-// disconnect the connection to union work and connect to merge
work
-followingWork = mergeJoinWork;
-rWork = (ReduceWork) mergeJoinWork.getMainWork();
+ int index = getFollowingWorkIndex(tezWork, unionWork, rs);
+ if (index != -1) {
+BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
+if (baseWork instanceof MergeJoinWork) {
+ MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
+ // disconnect the connection to union work and connect to merge
work
+ followingWork = mergeJoinWork;
+ rWork = (ReduceWork) mergeJoinWork.getMainWork();
+} else {
+ rWork = (ReduceWork) baseWork;
+}
} else {
-throw new SemanticException(Unknown work type found:
-+ baseWork.getClass().getCanonicalName());
+throw new SemanticException(Following work not found for the
reduce sink:
++ rs.getName());
}
} else {
rWork = (ReduceWork) followingWork;
@@ -403,19 +406,13 @@ public class GenTezWork implements NodeP
return null;
}
- private int getMergeIndex(TezWork tezWork, UnionWork unionWork,
ReduceSinkOperator rs) {
+ private int getFollowingWorkIndex(TezWork tezWork, UnionWork unionWork,
ReduceSinkOperator rs) {
int index = 0;
for (BaseWork baseWork : tezWork.getChildren(unionWork)) {
- if (baseWork instanceof MergeJoinWork) {
-MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
-int tag = mergeJoinWork.getMergeJoinOperator().getTagForOperator(rs);
-if (tag != -1) {
- return index;
-} else {
- index++;
-}
- } else {
+ if (tezWork.getEdgeProperty(unionWork,
baseWork).equals(TezEdgeProperty.EdgeType.CONTAINS)) {
index++;
+ } else {
+return index;
}
}
Added: hive
svn commit: r1646049 - in /hive/branches/branch-0.14: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/tez
Author: vikram
Date: Tue Dec 16 19:22:33 2014
New Revision: 1646049
URL: http://svn.apache.org/r1646049
Log:
HIVE-9055: Tez: union all followed by group by followed by another union all
gives error (Vikram Dixit K, reviewed by Prasanth J)
Added:
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/tez_multi_union.q
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/tez_multi_union.q.out
Modified:
hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
Modified:
hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties?rev=1646049r1=1646048r2=1646049view=diff
==
---
hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties
(original)
+++
hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties
Tue Dec 16 19:22:33 2014
@@ -253,6 +253,7 @@ minitez.query.files.shared=alter_merge_2
auto_sortmerge_join_8.q,\
auto_sortmerge_join_9.q
+
minitez.query.files=bucket_map_join_tez1.q,\
bucket_map_join_tez2.q,\
dynamic_partition_pruning.q,\
@@ -273,7 +274,8 @@ minitez.query.files=bucket_map_join_tez1
tez_union_group_by.q,\
tez_smb_main.q,\
tez_smb_1.q,\
- vectorized_dynamic_partition_pruning.q
+ vectorized_dynamic_partition_pruning.q,\
+ tez_multi_union.q
beeline.positive.exclude=add_part_exist.q,\
alter1.q,\
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java?rev=1646049r1=1646048r2=1646049view=diff
==
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
Tue Dec 16 19:22:33 2014
@@ -111,7 +111,7 @@ public class GenTezWork implements NodeP
// will result into a vertex with multiple FS or RS operators.
if (context.childToWorkMap.containsKey(operator)) {
// if we've seen both root and child, we can bail.
-
+
// clear out the mapjoin set. we don't need it anymore.
context.currentMapJoinOperators.clear();
@@ -349,17 +349,20 @@ public class GenTezWork implements NodeP
} else if (followingWork instanceof UnionWork) {
// this can only be possible if there is merge work followed by the
union
UnionWork unionWork = (UnionWork) followingWork;
- int index = getMergeIndex(tezWork, unionWork, rs);
- // guaranteed to be instance of MergeJoinWork if index is valid
- BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
- if (baseWork instanceof MergeJoinWork) {
-MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
-// disconnect the connection to union work and connect to merge
work
-followingWork = mergeJoinWork;
-rWork = (ReduceWork) mergeJoinWork.getMainWork();
+ int index = getFollowingWorkIndex(tezWork, unionWork, rs);
+ if (index != -1) {
+BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
+if (baseWork instanceof MergeJoinWork) {
+ MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
+ // disconnect the connection to union work and connect to merge
work
+ followingWork = mergeJoinWork;
+ rWork = (ReduceWork) mergeJoinWork.getMainWork();
+} else {
+ rWork = (ReduceWork) baseWork;
+}
} else {
-throw new SemanticException(Unknown work type found:
-+ baseWork.getClass().getCanonicalName());
+throw new SemanticException(Following work not found for the
reduce sink:
++ rs.getName());
}
} else {
rWork = (ReduceWork) followingWork;
@@ -403,19 +406,13 @@ public class GenTezWork implements NodeP
return null;
}
- private int getMergeIndex(TezWork tezWork, UnionWork unionWork,
ReduceSinkOperator rs) {
+ private int getFollowingWorkIndex(TezWork tezWork, UnionWork unionWork,
ReduceSinkOperator rs) {
int index = 0;
for (BaseWork baseWork : tezWork.getChildren(unionWork)) {
- if (baseWork instanceof MergeJoinWork) {
-MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
-int tag = mergeJoinWork.getMergeJoinOperator().getTagForOperator(rs);
-if (tag != -1) {
- return index;
-} else {
- index++;
-}
- } else
svn commit: r1616626 - in /hive/branches/tez: ./ ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/tools/
Author: vikram
Date: Thu Aug 7 23:22:08 2014
New Revision: 1616626
URL: http://svn.apache.org/r1616626
Log:
HIVE-7601: Bring up tez-branch upto the API changes from TEZ-1058, TEZ-1303,
TEZ-1346, TEZ-1041 (Gopal V via Vikram Dixit)
Modified:
hive/branches/tez/pom.xml
hive/branches/tez/ql/pom.xml
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HivePreWarmProcessor.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapTezProcessor.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceTezProcessor.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/tools/TezMergedLogicalInput.java
Modified: hive/branches/tez/pom.xml
URL:
http://svn.apache.org/viewvc/hive/branches/tez/pom.xml?rev=1616626r1=1616625r2=1616626view=diff
==
--- hive/branches/tez/pom.xml (original)
+++ hive/branches/tez/pom.xml Thu Aug 7 23:22:08 2014
@@ -144,7 +144,7 @@
stax.version1.0.1/stax.version
slf4j.version1.7.5/slf4j.version
ST4.version4.0.4/ST4.version
-tez.version0.5.0-incubating-SNAPSHOT/tez.version
+tez.version0.5.0-SNAPSHOT/tez.version
tempus-fugit.version1.1/tempus-fugit.version
snappy.version0.2/snappy.version
wadl-resourcedoc-doclet.version1.4/wadl-resourcedoc-doclet.version
Modified: hive/branches/tez/ql/pom.xml
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/pom.xml?rev=1616626r1=1616625r2=1616626view=diff
==
--- hive/branches/tez/ql/pom.xml (original)
+++ hive/branches/tez/ql/pom.xml Thu Aug 7 23:22:08 2014
@@ -297,6 +297,38 @@
/dependency
dependency
groupIdorg.apache.tez/groupId
+ artifactIdtez-runtime-internals/artifactId
+ version${tez.version}/version
+ optionaltrue/optional
+ exclusions
+exclusion
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-common/artifactId
+/exclusion
+exclusion
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-mapreduce-client-core/artifactId
+/exclusion
+exclusion
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-mapreduce-client-jobclient/artifactId
+/exclusion
+exclusion
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-mapreduce-client-common/artifactId
+/exclusion
+exclusion
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-hdfs/artifactId
+/exclusion
+ exclusion
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-yarn-client/artifactId
+ /exclusion
+ /exclusions
+/dependency
+dependency
+ groupIdorg.apache.tez/groupId
artifactIdtez-mapreduce/artifactId
version${tez.version}/version
optionaltrue/optional
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java?rev=1616626r1=1616625r2=1616626view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
Thu Aug 7 23:22:08 2014
@@ -31,6 +31,8 @@ import org.apache.tez.dag.api.EdgeManage
import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.api.events.InputReadErrorEvent;
+import com.google.common.collect.Multimap;
+
public class CustomPartitionEdge extends EdgeManager {
private static final Log LOG =
LogFactory.getLog(CustomPartitionEdge.class.getName());
@@ -39,9 +41,11 @@ public class CustomPartitionEdge extends
EdgeManagerContext context = null;
// used by the framework at runtime. initialize is the real initializer at
runtime
- public CustomPartitionEdge() {
+ public CustomPartitionEdge(EdgeManagerContext context) {
+super(context);
}
+
@Override
public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) {
return
svn commit: r1616627 - in /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez: DagUtils.java ObjectCache.java tools/InputMerger.java
Author: vikram
Date: Thu Aug 7 23:23:23 2014
New Revision: 1616627
URL: http://svn.apache.org/r1616627
Log:
HIVE-7639: Bring tez-branch upto api changes in TEZ-1379, TEZ-1057, TEZ-1382
(Gopal V via Vikram Dixit)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/tools/InputMerger.java
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1616627r1=1616626r2=1616627view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Thu Aug 7 23:23:23 2014
@@ -81,6 +81,8 @@ import org.apache.hadoop.yarn.util.Conve
import org.apache.hadoop.yarn.util.Records;
import org.apache.tez.client.PreWarmContext;
import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.DataSourceDescriptor;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeManagerDescriptor;
import org.apache.tez.dag.api.EdgeProperty;
@@ -302,7 +304,7 @@ public class DagUtils {
String keyClass = conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS);
String valClass =
conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS);
String partitionerClassName = conf.get(mapred.partitioner.class);
-Configuration partitionerConf;
+MapString, String partitionerConf;
EdgeType edgeType = edgeProp.getEdgeType();
switch (edgeType) {
@@ -352,12 +354,12 @@ public class DagUtils {
* a base configuration to extract relevant properties
* @return
*/
- private Configuration createPartitionerConf(String partitionerClassName,
+ private MapString, String createPartitionerConf(String
partitionerClassName,
Configuration baseConf) {
-Configuration partitionerConf = new Configuration(false);
-partitionerConf.set(mapred.partitioner.class, partitionerClassName);
+MapString, String partitionerConf = new HashMapString, String();
+partitionerConf.put(mapred.partitioner.class, partitionerClassName);
if (baseConf.get(mapreduce.totalorderpartitioner.path) != null) {
- partitionerConf.set(mapreduce.totalorderpartitioner.path,
+ partitionerConf.put(mapreduce.totalorderpartitioner.path,
baseConf.get(mapreduce.totalorderpartitioner.path));
}
return partitionerConf;
@@ -491,8 +493,8 @@ public class DagUtils {
mrInput = MRHelpers.createMRInputPayload(serializedConf, null);
}
map.addDataSource(alias,
-new InputDescriptor(MRInputLegacy.class.getName()).
-setUserPayload(mrInput), new
InputInitializerDescriptor(amSplitGeneratorClass.getName()).setUserPayload(mrInput));
+new DataSourceDescriptor(new
InputDescriptor(MRInputLegacy.class.getName()).
+setUserPayload(mrInput), new
InputInitializerDescriptor(amSplitGeneratorClass.getName()).setUserPayload(mrInput),null));
MapString, LocalResource localResources = new HashMapString,
LocalResource();
localResources.put(getBaseName(appJarLr), appJarLr);
@@ -946,9 +948,9 @@ public class DagUtils {
// final vertices need to have at least one output
if (!hasChildren) {
- v.addDataSink(out_+work.getName(),
+ v.addDataSink(out_+work.getName(), new DataSinkDescriptor(
new OutputDescriptor(MROutput.class.getName())
- .setUserPayload(MRHelpers.createUserPayloadFromConf(conf)), null);
+ .setUserPayload(MRHelpers.createUserPayloadFromConf(conf)), null,
null));
}
return v;
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java?rev=1616627r1=1616626r2=1616627view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
Thu Aug 7 23:23:23 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec.t
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.tez.runtime.common.objectregistry.ObjectLifeCycle;
import org.apache.tez.runtime.common.objectregistry.ObjectRegistry;
import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
@@ -36,7 +35,7 @@ public class ObjectCache implements org.
@Override
public void cache(String key, Object value
svn commit: r1616904 - in /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez: CustomPartitionEdge.java DagUtils.java HiveSplitGenerator.java TezSessionState.java
Author: vikram
Date: Fri Aug 8 23:23:55 2014
New Revision: 1616904
URL: http://svn.apache.org/r1616904
Log:
Bring tez-branch up-to the API changes made by TEZ-1372 (Gopal V via Vikram
Dixit)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java?rev=1616904r1=1616903r2=1616904view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
Fri Aug 8 23:23:55 2014
@@ -43,6 +43,7 @@ public class CustomPartitionEdge extends
// used by the framework at runtime. initialize is the real initializer at
runtime
public CustomPartitionEdge(EdgeManagerContext context) {
super(context);
+this.context = context;
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1616904r1=1616903r2=1616904view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Fri Aug 8 23:23:55 2014
@@ -79,7 +79,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.tez.client.PreWarmContext;
+import org.apache.tez.client.PreWarmVertex;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.DataSinkDescriptor;
import org.apache.tez.dag.api.DataSourceDescriptor;
@@ -380,6 +380,15 @@ public class DagUtils {
}
/*
+ * Helper to setup default environment for a task in YARN.
+ */
+ private MapString, String getContainerEnvironment(Configuration conf,
boolean isMap) {
+MapString, String environment = new HashMapString, String();
+MRHelpers.updateEnvironmentForMRTasks(conf, environment, isMap);
+return environment;
+ }
+
+ /*
* Helper to determine what java options to use for the containers
* Falls back to Map-reduces map java opts if no tez specific options
* are set
@@ -449,7 +458,7 @@ public class DagUtils {
// is HiveInputFormat
if (inputFormatClass == HiveInputFormat.class) {
useTezGroupedSplits = true;
-conf.setClass(mapred.input.format.class,
TezGroupedSplitsInputFormat.class, InputFormat.class);
+conf.setClass(mapred.input.format.class, HiveInputFormat.class,
InputFormat.class);
}
}
@@ -476,9 +485,7 @@ public class DagUtils {
map = new Vertex(mapWork.getName(),
new ProcessorDescriptor(MapTezProcessor.class.getName()).
setUserPayload(serializedConf), numTasks, getContainerResource(conf));
-MapString, String environment = new HashMapString, String();
-MRHelpers.updateEnvironmentForMRTasks(conf, environment, true);
-map.setTaskEnvironment(environment);
+map.setTaskEnvironment(getContainerEnvironment(conf, true));
map.setTaskLaunchCmdOpts(getContainerJavaOpts(conf));
assert mapWork.getAliasToWork().keySet().size() == 1;
@@ -487,10 +494,9 @@ public class DagUtils {
byte[] mrInput = null;
if (useTezGroupedSplits) {
- mrInput = MRHelpers.createMRInputPayloadWithGrouping(serializedConf,
- HiveInputFormat.class.getName());
+ mrInput = MRHelpers.createMRInputPayloadWithGrouping(serializedConf);
} else {
- mrInput = MRHelpers.createMRInputPayload(serializedConf, null);
+ mrInput = MRHelpers.createMRInputPayload(serializedConf);
}
map.addDataSource(alias,
new DataSourceDescriptor(new
InputDescriptor(MRInputLegacy.class.getName()).
@@ -550,11 +556,7 @@ public class DagUtils {
reduceWork.isAutoReduceParallelism() ?
reduceWork.getMaxReduceTasks() : reduceWork
.getNumReduceTasks(), getContainerResource(conf));
-MapString, String environment = new HashMapString, String();
-
-MRHelpers.updateEnvironmentForMRTasks(conf, environment, false);
-reducer.setTaskEnvironment(environment);
-
+reducer.setTaskEnvironment(getContainerEnvironment(conf, false));
reducer.setTaskLaunchCmdOpts
svn commit: r1616905 - in /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez: CustomPartitionEdge.java DagUtils.java HiveSplitGenerator.java TezSessionState.java
Author: vikram
Date: Fri Aug 8 23:28:21 2014
New Revision: 1616905
URL: http://svn.apache.org/r1616905
Log:
HIVE-7656: Bring tez-branch up-to the API changes made by TEZ-1372 (Gopal V via
Vikram Dixit)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java?rev=1616905r1=1616904r2=1616905view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
Fri Aug 8 23:28:21 2014
@@ -43,7 +43,6 @@ public class CustomPartitionEdge extends
// used by the framework at runtime. initialize is the real initializer at
runtime
public CustomPartitionEdge(EdgeManagerContext context) {
super(context);
-this.context = context;
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1616905r1=1616904r2=1616905view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Fri Aug 8 23:28:21 2014
@@ -79,7 +79,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.tez.client.PreWarmVertex;
+import org.apache.tez.client.PreWarmContext;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.DataSinkDescriptor;
import org.apache.tez.dag.api.DataSourceDescriptor;
@@ -380,15 +380,6 @@ public class DagUtils {
}
/*
- * Helper to setup default environment for a task in YARN.
- */
- private MapString, String getContainerEnvironment(Configuration conf,
boolean isMap) {
-MapString, String environment = new HashMapString, String();
-MRHelpers.updateEnvironmentForMRTasks(conf, environment, isMap);
-return environment;
- }
-
- /*
* Helper to determine what java options to use for the containers
* Falls back to Map-reduces map java opts if no tez specific options
* are set
@@ -458,7 +449,7 @@ public class DagUtils {
// is HiveInputFormat
if (inputFormatClass == HiveInputFormat.class) {
useTezGroupedSplits = true;
-conf.setClass(mapred.input.format.class, HiveInputFormat.class,
InputFormat.class);
+conf.setClass(mapred.input.format.class,
TezGroupedSplitsInputFormat.class, InputFormat.class);
}
}
@@ -485,7 +476,9 @@ public class DagUtils {
map = new Vertex(mapWork.getName(),
new ProcessorDescriptor(MapTezProcessor.class.getName()).
setUserPayload(serializedConf), numTasks, getContainerResource(conf));
-map.setTaskEnvironment(getContainerEnvironment(conf, true));
+MapString, String environment = new HashMapString, String();
+MRHelpers.updateEnvironmentForMRTasks(conf, environment, true);
+map.setTaskEnvironment(environment);
map.setTaskLaunchCmdOpts(getContainerJavaOpts(conf));
assert mapWork.getAliasToWork().keySet().size() == 1;
@@ -494,9 +487,10 @@ public class DagUtils {
byte[] mrInput = null;
if (useTezGroupedSplits) {
- mrInput = MRHelpers.createMRInputPayloadWithGrouping(serializedConf);
+ mrInput = MRHelpers.createMRInputPayloadWithGrouping(serializedConf,
+ HiveInputFormat.class.getName());
} else {
- mrInput = MRHelpers.createMRInputPayload(serializedConf);
+ mrInput = MRHelpers.createMRInputPayload(serializedConf, null);
}
map.addDataSource(alias,
new DataSourceDescriptor(new
InputDescriptor(MRInputLegacy.class.getName()).
@@ -556,7 +550,11 @@ public class DagUtils {
reduceWork.isAutoReduceParallelism() ?
reduceWork.getMaxReduceTasks() : reduceWork
.getNumReduceTasks(), getContainerResource(conf));
-reducer.setTaskEnvironment(getContainerEnvironment(conf, false));
+MapString, String environment = new HashMapString, String();
+
+MRHelpers.updateEnvironmentForMRTasks(conf, environment, false);
+reducer.setTaskEnvironment(environment);
+
reducer.setTaskLaunchCmdOpts
svn commit: r1616906 - in /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez: CustomPartitionEdge.java DagUtils.java HiveSplitGenerator.java TezSessionState.java
Author: vikram
Date: Fri Aug 8 23:29:23 2014
New Revision: 1616906
URL: http://svn.apache.org/r1616906
Log:
HIVE-7656: Bring tez-branch up-to the API changes made by TEZ-1372 (Gopal V via
Vikram Dixit)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java?rev=1616906r1=1616905r2=1616906view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
Fri Aug 8 23:29:23 2014
@@ -43,6 +43,7 @@ public class CustomPartitionEdge extends
// used by the framework at runtime. initialize is the real initializer at
runtime
public CustomPartitionEdge(EdgeManagerContext context) {
super(context);
+this.context = context;
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1616906r1=1616905r2=1616906view=diff
==
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Fri Aug 8 23:29:23 2014
@@ -79,7 +79,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.tez.client.PreWarmContext;
+import org.apache.tez.client.PreWarmVertex;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.DataSinkDescriptor;
import org.apache.tez.dag.api.DataSourceDescriptor;
@@ -380,6 +380,15 @@ public class DagUtils {
}
/*
+ * Helper to setup default environment for a task in YARN.
+ */
+ private MapString, String getContainerEnvironment(Configuration conf,
boolean isMap) {
+MapString, String environment = new HashMapString, String();
+MRHelpers.updateEnvironmentForMRTasks(conf, environment, isMap);
+return environment;
+ }
+
+ /*
* Helper to determine what java options to use for the containers
* Falls back to Map-reduces map java opts if no tez specific options
* are set
@@ -449,7 +458,7 @@ public class DagUtils {
// is HiveInputFormat
if (inputFormatClass == HiveInputFormat.class) {
useTezGroupedSplits = true;
-conf.setClass(mapred.input.format.class,
TezGroupedSplitsInputFormat.class, InputFormat.class);
+conf.setClass(mapred.input.format.class, HiveInputFormat.class,
InputFormat.class);
}
}
@@ -476,9 +485,7 @@ public class DagUtils {
map = new Vertex(mapWork.getName(),
new ProcessorDescriptor(MapTezProcessor.class.getName()).
setUserPayload(serializedConf), numTasks, getContainerResource(conf));
-MapString, String environment = new HashMapString, String();
-MRHelpers.updateEnvironmentForMRTasks(conf, environment, true);
-map.setTaskEnvironment(environment);
+map.setTaskEnvironment(getContainerEnvironment(conf, true));
map.setTaskLaunchCmdOpts(getContainerJavaOpts(conf));
assert mapWork.getAliasToWork().keySet().size() == 1;
@@ -487,10 +494,9 @@ public class DagUtils {
byte[] mrInput = null;
if (useTezGroupedSplits) {
- mrInput = MRHelpers.createMRInputPayloadWithGrouping(serializedConf,
- HiveInputFormat.class.getName());
+ mrInput = MRHelpers.createMRInputPayloadWithGrouping(serializedConf);
} else {
- mrInput = MRHelpers.createMRInputPayload(serializedConf, null);
+ mrInput = MRHelpers.createMRInputPayload(serializedConf);
}
map.addDataSource(alias,
new DataSourceDescriptor(new
InputDescriptor(MRInputLegacy.class.getName()).
@@ -550,11 +556,7 @@ public class DagUtils {
reduceWork.isAutoReduceParallelism() ?
reduceWork.getMaxReduceTasks() : reduceWork
.getNumReduceTasks(), getContainerResource(conf));
-MapString, String environment = new HashMapString, String();
-
-MRHelpers.updateEnvironmentForMRTasks(conf, environment, false);
-reducer.setTaskEnvironment(environment);
-
+reducer.setTaskEnvironment(getContainerEnvironment(conf, false));
reducer.setTaskLaunchCmdOpts
svn commit: r1617801 - in /hive/trunk: itests/src/test/resources/testconfiguration.properties pom.xml
Author: vikram Date: Wed Aug 13 19:07:59 2014 New Revision: 1617801 URL: http://svn.apache.org/r1617801 Log: HIVE-7477: Upgrade hive to use tez 0.4.1 (Vikram Dixit, reviewed by Ashutosh Chouhan, Navis Ryu) Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties hive/trunk/pom.xml Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1617801r1=1617800r2=1617801view=diff == --- hive/trunk/itests/src/test/resources/testconfiguration.properties (original) +++ hive/trunk/itests/src/test/resources/testconfiguration.properties Wed Aug 13 19:07:59 2014 @@ -87,7 +87,6 @@ minitez.query.files.shared=alter_merge_2 merge1.q,\ merge2.q,\ metadata_only_queries.q,\ - metadataonly1.q,\ optimize_nullscan.q,\ orc_analyze.q,\ orc_merge1.q,\ Modified: hive/trunk/pom.xml URL: http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1617801r1=1617800r2=1617801view=diff == --- hive/trunk/pom.xml (original) +++ hive/trunk/pom.xml Wed Aug 13 19:07:59 2014 @@ -147,7 +147,7 @@ slf4j.version1.7.5/slf4j.version ST4.version4.0.4/ST4.version super-csv.version2.2.0/super-csv.version -tez.version0.4.0-incubating/tez.version +tez.version0.4.1-incubating/tez.version tempus-fugit.version1.1/tempus-fugit.version snappy.version0.2/snappy.version wadl-resourcedoc-doclet.version1.4/wadl-resourcedoc-doclet.version
svn commit: r1635016 - /hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
Author: vikram
Date: Wed Oct 29 00:41:41 2014
New Revision: 1635016
URL: http://svn.apache.org/r1635016
Log:
HIVE-8597: SMB join small table side should use the same set of serialized
payloads across tasks (Siddharth Seth via Vikram Dixit)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java?rev=1635016r1=1635015r2=1635016view=diff
==
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
Wed Oct 29 00:41:41 2014
@@ -32,6 +32,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import com.google.common.collect.LinkedListMultimap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -109,7 +110,6 @@ public class CustomPartitionVertex exten
VertexManagerPluginContext context;
private InputConfigureVertexTasksEvent configureVertexTaskEvent;
- private ListInputDataInformationEvent dataInformationEvents;
private int numBuckets = -1;
private Configuration conf = null;
private final SplitGrouper grouper = new SplitGrouper();
@@ -223,8 +223,6 @@ public class CustomPartitionVertex exten
configureVertexTaskEvent = cEvent;
LOG.info(Configure task for input name: + inputName + num tasks:
+ configureVertexTaskEvent.getNumTasks());
-dataInformationEvents =
-
Lists.newArrayListWithCapacity(configureVertexTaskEvent.getNumTasks());
}
if (event instanceof InputUpdatePayloadEvent) {
// this event can never occur. If it does, fail.
@@ -232,7 +230,6 @@ public class CustomPartitionVertex exten
} else if (event instanceof InputDataInformationEvent) {
dataInformationEventSeen = true;
InputDataInformationEvent diEvent = (InputDataInformationEvent) event;
-dataInformationEvents.add(diEvent);
FileSplit fileSplit;
try {
fileSplit = getFileSplitFromEvent(diEvent);
@@ -341,16 +338,26 @@ public class CustomPartitionVertex exten
+ multi mr inputs. + bucketToTaskMap);
Integer[] numSplitsForTask = new Integer[taskCount];
+
+MultimapInteger, ByteBuffer bucketToSerializedSplitMap =
LinkedListMultimap.create();
+
+// Create the list of serialized splits for each bucket.
for (EntryInteger, CollectionInputSplit entry :
bucketToGroupedSplitMap.asMap().entrySet()) {
+ for (InputSplit split : entry.getValue()) {
+MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(split);
+ByteBuffer bs = serializedSplit.toByteString().asReadOnlyByteBuffer();
+bucketToSerializedSplitMap.put(entry.getKey(), bs);
+ }
+}
+
+for (EntryInteger, CollectionByteBuffer entry :
bucketToSerializedSplitMap.asMap().entrySet()) {
CollectionInteger destTasks = bucketToTaskMap.get(entry.getKey());
for (Integer task : destTasks) {
int count = 0;
-for (InputSplit split : entry.getValue()) {
+for (ByteBuffer buf : entry.getValue()) {
count++;
- MRSplitProto serializedSplit =
MRInputHelpers.createSplitProto(split);
InputDataInformationEvent diEvent =
- InputDataInformationEvent.createWithSerializedPayload(task,
serializedSplit
- .toByteString().asReadOnlyByteBuffer());
+ InputDataInformationEvent.createWithSerializedPayload(count,
buf);
diEvent.setTargetIndex(task);
taskEvents.add(diEvent);
}
@@ -472,6 +479,7 @@ public class CustomPartitionVertex exten
context.setVertexParallelism(taskCount, VertexLocationHint.create(grouper
.createTaskLocationHints(finalSplits.toArray(new
InputSplit[finalSplits.size()]))), emMap,
rootInputSpecUpdate);
+finalSplits.clear();
}
UserPayload getBytePayload(MultimapInteger, Integer routingTable) throws
IOException {
svn commit: r1624170 - /hive/branches/llap/
Author: vikram Date: Wed Sep 10 23:50:49 2014 New Revision: 1624170 URL: http://svn.apache.org/r1624170 Log: Branch for LLAP development Added: hive/branches/llap/ (props changed) - copied from r1624169, hive/trunk/ Propchange: hive/branches/llap/ -- reviewboard:url = https://reviews.apache.org Propchange: hive/branches/llap/ -- --- svn:ignore (added) +++ svn:ignore Wed Sep 10 23:50:49 2014 @@ -0,0 +1,11 @@ +datanucleus.log +eclipse-build +target +.arc_jira_lib +.classpath* +.externalToolBuilders +.project +.settings +*.launch +*~ +metastore_db Propchange: hive/branches/llap/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Wed Sep 10 23:50:49 2014 @@ -0,0 +1,3 @@ +/hive/branches/branch-0.11:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184 +/hive/branches/tez:1494760-1622766 +/hive/branches/vectorization:1466908-1527856
svn commit: r1627445 - /hive/branches/branch-0.14/
Author: vikram Date: Thu Sep 25 02:14:15 2014 New Revision: 1627445 URL: http://svn.apache.org/r1627445 Log: Creation of branch 0.14 Added: hive/branches/branch-0.14/ (props changed) - copied from r1627444, hive/trunk/ Propchange: hive/branches/branch-0.14/ -- reviewboard:url = https://reviews.apache.org Propchange: hive/branches/branch-0.14/ -- --- svn:ignore (added) +++ svn:ignore Thu Sep 25 02:14:15 2014 @@ -0,0 +1,11 @@ +datanucleus.log +eclipse-build +target +.arc_jira_lib +.classpath* +.externalToolBuilders +.project +.settings +*.launch +*~ +metastore_db Propchange: hive/branches/branch-0.14/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Thu Sep 25 02:14:15 2014 @@ -0,0 +1,4 @@ +/hive/branches/branch-0.11:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184 +/hive/branches/cbo:1605012-1627125 +/hive/branches/tez:1494760-1622766 +/hive/branches/vectorization:1466908-1527856
svn commit: r1628349 - in /hive/branches/branch-0.14: accumulo-handler/pom.xml pom.xml
Author: vikram
Date: Tue Sep 30 01:40:36 2014
New Revision: 1628349
URL: http://svn.apache.org/r1628349
Log:
HIVE-8257: Accumulo introduces old hadoop-client dependency (Josh Esler via
Vikram Dixit K)
Modified:
hive/branches/branch-0.14/accumulo-handler/pom.xml
hive/branches/branch-0.14/pom.xml
Modified: hive/branches/branch-0.14/accumulo-handler/pom.xml
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/accumulo-handler/pom.xml?rev=1628349r1=1628348r2=1628349view=diff
==
--- hive/branches/branch-0.14/accumulo-handler/pom.xml (original)
+++ hive/branches/branch-0.14/accumulo-handler/pom.xml Tue Sep 30 01:40:36 2014
@@ -112,6 +112,12 @@
dependencies
dependency
groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-client/artifactId
+ version${hadoop-20S.version}/version
+ optionaltrue/optional
+/dependency
+dependency
+ groupIdorg.apache.hadoop/groupId
artifactIdhadoop-core/artifactId
version${hadoop-20S.version}/version
optionaltrue/optional
@@ -123,6 +129,12 @@
dependencies
dependency
groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-client/artifactId
+ version${hadoop-23.version}/version
+ optionaltrue/optional
+/dependency
+dependency
+ groupIdorg.apache.hadoop/groupId
artifactIdhadoop-common/artifactId
version${hadoop-23.version}/version
optionaltrue/optional
Modified: hive/branches/branch-0.14/pom.xml
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/pom.xml?rev=1628349r1=1628348r2=1628349view=diff
==
--- hive/branches/branch-0.14/pom.xml (original)
+++ hive/branches/branch-0.14/pom.xml Tue Sep 30 01:40:36 2014
@@ -969,6 +969,11 @@
dependencies
dependency
groupIdorg.apache.hadoop/groupId
+artifactIdhadoop-client/artifactId
+version${hadoop-20S.version}/version
+ /dependency
+ dependency
+groupIdorg.apache.hadoop/groupId
artifactIdhadoop-core/artifactId
version${hadoop-20S.version}/version
/dependency
@@ -1011,6 +1016,11 @@
dependencies
dependency
groupIdorg.apache.hadoop/groupId
+artifactIdhadoop-client/artifactId
+version${hadoop-23.version}/version
+ /dependency
+ dependency
+groupIdorg.apache.hadoop/groupId
artifactIdhadoop-common/artifactId
version${hadoop-23.version}/version
/dependency
svn commit: r1628352 - in /hive/trunk: accumulo-handler/pom.xml pom.xml
Author: vikram
Date: Tue Sep 30 01:44:55 2014
New Revision: 1628352
URL: http://svn.apache.org/r1628352
Log:
HIVE-8257: Accumulo introduces old hadoop-client dependency (Josh Esler via
Vikram Dixit K)
Modified:
hive/trunk/accumulo-handler/pom.xml
hive/trunk/pom.xml
Modified: hive/trunk/accumulo-handler/pom.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/pom.xml?rev=1628352r1=1628351r2=1628352view=diff
==
--- hive/trunk/accumulo-handler/pom.xml (original)
+++ hive/trunk/accumulo-handler/pom.xml Tue Sep 30 01:44:55 2014
@@ -112,6 +112,12 @@
dependencies
dependency
groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-client/artifactId
+ version${hadoop-20S.version}/version
+ optionaltrue/optional
+/dependency
+dependency
+ groupIdorg.apache.hadoop/groupId
artifactIdhadoop-core/artifactId
version${hadoop-20S.version}/version
optionaltrue/optional
@@ -123,6 +129,12 @@
dependencies
dependency
groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-client/artifactId
+ version${hadoop-23.version}/version
+ optionaltrue/optional
+/dependency
+dependency
+ groupIdorg.apache.hadoop/groupId
artifactIdhadoop-common/artifactId
version${hadoop-23.version}/version
optionaltrue/optional
Modified: hive/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1628352r1=1628351r2=1628352view=diff
==
--- hive/trunk/pom.xml (original)
+++ hive/trunk/pom.xml Tue Sep 30 01:44:55 2014
@@ -969,6 +969,11 @@
dependencies
dependency
groupIdorg.apache.hadoop/groupId
+artifactIdhadoop-client/artifactId
+version${hadoop-20S.version}/version
+ /dependency
+ dependency
+groupIdorg.apache.hadoop/groupId
artifactIdhadoop-core/artifactId
version${hadoop-20S.version}/version
/dependency
@@ -1011,6 +1016,11 @@
dependencies
dependency
groupIdorg.apache.hadoop/groupId
+artifactIdhadoop-client/artifactId
+version${hadoop-23.version}/version
+ /dependency
+ dependency
+groupIdorg.apache.hadoop/groupId
artifactIdhadoop-common/artifactId
version${hadoop-23.version}/version
/dependency
svn commit: r1628512 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/tez/ java/org/apache/hadoop/hive/ql/session/ test/org/apache/hadoop/hive/ql/exec/tez/
Author: vikram
Date: Tue Sep 30 18:36:56 2014
New Revision: 1628512
URL: http://svn.apache.org/r1628512
Log:
HIVE-7857: Hive query fails after Tez session times out (Vikram Dixit K,
reviewed by Gunther Hagleitner)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java?rev=1628512r1=1628511r2=1628512view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
Tue Sep 30 18:36:56 2014
@@ -78,7 +78,7 @@ public class TezJobMonitor {
try {
for (TezSessionState s: TezSessionState.getOpenSessions()) {
System.err.println(Shutting down tez session.);
-TezSessionPoolManager.getInstance().close(s);
+TezSessionPoolManager.getInstance().close(s, false);
}
} catch (Exception e) {
// ignore
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java?rev=1628512r1=1628511r2=1628512view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
Tue Sep 30 18:36:56 2014
@@ -168,10 +168,10 @@ public class TezSessionPoolManager {
// session in the SessionState
}
- public void close(TezSessionState tezSessionState) throws Exception {
+ public void close(TezSessionState tezSessionState, boolean keepTmpDir)
throws Exception {
LOG.info(Closing tez session default? + tezSessionState.isDefault());
if (!tezSessionState.isDefault()) {
- tezSessionState.close(false);
+ tezSessionState.close(keepTmpDir);
}
}
@@ -262,24 +262,24 @@ public class TezSessionPoolManager {
}
if (session != null) {
- close(session);
+ close(session, false);
}
return getSession(conf, doOpen, forceCreate);
}
- public void closeAndOpen(TezSessionState sessionState, HiveConf conf)
+ public void closeAndOpen(TezSessionState sessionState, HiveConf conf,
boolean keepTmpDir)
throws Exception {
-closeAndOpen(sessionState, conf, null);
+closeAndOpen(sessionState, conf, null, keepTmpDir);
}
public void closeAndOpen(TezSessionState sessionState, HiveConf conf,
- String[] additionalFiles) throws Exception {
+ String[] additionalFiles, boolean keepTmpDir) throws Exception {
HiveConf sessionConf = sessionState.getConf();
if (sessionConf != null sessionConf.get(tez.queue.name) != null) {
conf.set(tez.queue.name, sessionConf.get(tez.queue.name));
}
-close(sessionState);
+close(sessionState, keepTmpDir);
sessionState.open(conf, additionalFiles);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1628512r1=1628511r2=1628512view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Tue
Sep 30 18:36:56 2014
@@ -370,7 +370,7 @@ public class TezTask extends TaskTezWor
console.printInfo(Tez session was closed. Reopening...);
// close the old one, but keep the tmp files around
- TezSessionPoolManager.getInstance().closeAndOpen(sessionState,
this.conf, inputOutputJars);
+ TezSessionPoolManager.getInstance().closeAndOpen(sessionState,
this.conf, inputOutputJars, true);
console.printInfo(Session re-established.);
dagClient = sessionState.getSession().submitDAG(dag);
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1628512r1=1628511r2=1628512view=diff
svn commit: r1628511 - in /hive/branches/branch-0.14/ql/src: java/org/apache/hadoop/hive/ql/exec/tez/ java/org/apache/hadoop/hive/ql/session/ test/org/apache/hadoop/hive/ql/exec/tez/
Author: vikram
Date: Tue Sep 30 18:34:55 2014
New Revision: 1628511
URL: http://svn.apache.org/r1628511
Log:
HIVE-7857: Hive query fails after Tez session times out (Vikram Dixit K,
reviewed by Gunther Hagleitner)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java?rev=1628511r1=1628510r2=1628511view=diff
==
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
Tue Sep 30 18:34:55 2014
@@ -78,7 +78,7 @@ public class TezJobMonitor {
try {
for (TezSessionState s: TezSessionState.getOpenSessions()) {
System.err.println(Shutting down tez session.);
-TezSessionPoolManager.getInstance().close(s);
+TezSessionPoolManager.getInstance().close(s, false);
}
} catch (Exception e) {
// ignore
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java?rev=1628511r1=1628510r2=1628511view=diff
==
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
Tue Sep 30 18:34:55 2014
@@ -168,10 +168,10 @@ public class TezSessionPoolManager {
// session in the SessionState
}
- public void close(TezSessionState tezSessionState) throws Exception {
+ public void close(TezSessionState tezSessionState, boolean keepTmpDir)
throws Exception {
LOG.info(Closing tez session default? + tezSessionState.isDefault());
if (!tezSessionState.isDefault()) {
- tezSessionState.close(false);
+ tezSessionState.close(keepTmpDir);
}
}
@@ -262,24 +262,24 @@ public class TezSessionPoolManager {
}
if (session != null) {
- close(session);
+ close(session, false);
}
return getSession(conf, doOpen, forceCreate);
}
- public void closeAndOpen(TezSessionState sessionState, HiveConf conf)
+ public void closeAndOpen(TezSessionState sessionState, HiveConf conf,
boolean keepTmpDir)
throws Exception {
-closeAndOpen(sessionState, conf, null);
+closeAndOpen(sessionState, conf, null, keepTmpDir);
}
public void closeAndOpen(TezSessionState sessionState, HiveConf conf,
- String[] additionalFiles) throws Exception {
+ String[] additionalFiles, boolean keepTmpDir) throws Exception {
HiveConf sessionConf = sessionState.getConf();
if (sessionConf != null sessionConf.get(tez.queue.name) != null) {
conf.set(tez.queue.name, sessionConf.get(tez.queue.name));
}
-close(sessionState);
+close(sessionState, keepTmpDir);
sessionState.open(conf, additionalFiles);
}
}
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1628511r1=1628510r2=1628511view=diff
==
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
Tue Sep 30 18:34:55 2014
@@ -370,7 +370,7 @@ public class TezTask extends TaskTezWor
console.printInfo(Tez session was closed. Reopening...);
// close the old one, but keep the tmp files around
- TezSessionPoolManager.getInstance().closeAndOpen(sessionState,
this.conf, inputOutputJars);
+ TezSessionPoolManager.getInstance().closeAndOpen(sessionState,
this.conf, inputOutputJars, true);
console.printInfo(Session re-established.);
dagClient = sessionState.getSession().submitDAG(dag);
Modified:
hive/branches
svn commit: r1631010 - /hive/trunk/itests/src/test/resources/testconfiguration.properties
Author: vikram Date: Sat Oct 11 01:30:09 2014 New Revision: 1631010 URL: http://svn.apache.org/r1631010 Log: HIVE-8432: Remove tez_smb_1 from tests. (Vikram Dixit K) Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1631010r1=1631009r2=1631010view=diff == --- hive/trunk/itests/src/test/resources/testconfiguration.properties (original) +++ hive/trunk/itests/src/test/resources/testconfiguration.properties Sat Oct 11 01:30:09 2014 @@ -241,7 +241,6 @@ minitez.query.files=bucket_map_join_tez1 tez_union_decimal.q,\ tez_union_group_by.q,\ tez_smb_main.q,\ - tez_smb_1.q,\ vectorized_dynamic_partition_pruning.q beeline.positive.exclude=add_part_exist.q,\
svn commit: r1631566 - in /hive/trunk: itests/src/test/resources/testconfiguration.properties ql/src/test/queries/clientpositive/tez_smb_1.q ql/src/test/results/clientpositive/tez/tez_smb_1.q.out
Author: vikram Date: Mon Oct 13 21:38:43 2014 New Revision: 1631566 URL: http://svn.apache.org/r1631566 Log: HIVE-8443: Disable tez_smb_1 for mapreduce and prevent from test hang. (Vikram Dixit K reviewed by Gunther Hagleitner and Jason Dere) Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties hive/trunk/ql/src/test/queries/clientpositive/tez_smb_1.q hive/trunk/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1631566r1=1631565r2=1631566view=diff == --- hive/trunk/itests/src/test/resources/testconfiguration.properties (original) +++ hive/trunk/itests/src/test/resources/testconfiguration.properties Mon Oct 13 21:38:43 2014 @@ -241,6 +241,7 @@ minitez.query.files=bucket_map_join_tez1 tez_union_decimal.q,\ tez_union_group_by.q,\ tez_smb_main.q,\ + tez_smb_1.q,\ vectorized_dynamic_partition_pruning.q beeline.positive.exclude=add_part_exist.q,\ Modified: hive/trunk/ql/src/test/queries/clientpositive/tez_smb_1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/tez_smb_1.q?rev=1631566r1=1631565r2=1631566view=diff == --- hive/trunk/ql/src/test/queries/clientpositive/tez_smb_1.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/tez_smb_1.q Mon Oct 13 21:38:43 2014 @@ -33,6 +33,3 @@ set hive.auto.convert.join.noconditional explain select count(*) from tab s1 join tab s3 on s1.key=s3.key; -select s1.key, s1.value, s3.value from tab s1 join tab s3 on s1.key=s3.key; -select count(*) from tab s2; - Modified: hive/trunk/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out?rev=1631566r1=1631565r2=1631566view=diff == --- hive/trunk/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out Mon Oct 13 21:38:43 2014 @@ -186,504 +186,3 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select s1.key, s1.value, s3.value from tab s1 join tab s3 on s1.key=s3.key -PREHOOK: type: QUERY -PREHOOK: Input: default@tab -PREHOOK: Input: default@tab@ds=2008-04-08 - A masked pattern was here -POSTHOOK: query: select s1.key, s1.value, s3.value from tab s1 join tab s3 on s1.key=s3.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@tab -POSTHOOK: Input: default@tab@ds=2008-04-08 - A masked pattern was here -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -2 val_2 val_2 -4 val_4 val_4 -8 val_8 val_8 -20 val_20 val_20 -24 val_24 val_24 -24 val_24 val_24 -24 val_24 val_24 -24 val_24 val_24 -26 val_26 val_26 -26 val_26 val_26 -26 val_26 val_26 -26 val_26 val_26 -28 val_28 val_28 -42 val_42 val_42 -42 val_42 val_42 -42 val_42 val_42 -42 val_42 val_42 -44 val_44 val_44 -64 val_64 val_64 -66 val_66 val_66 -80 val_80 val_80 -82 val_82 val_82 -84 val_84 val_84 -84 val_84 val_84 -84 val_84 val_84 -84 val_84 val_84 -86 val_86 val_86 -114val_114 val_114 -116val_116 val_116 -118val_118 val_118 -118val_118 val_118 -118val_118 val_118 -118val_118 val_118 -134val_134 val_134 -134val_134 val_134 -134val_134 val_134 -134val_134 val_134 -136val_136 val_136 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -138val_138 val_138 -150val_150 val_150 -152val_152 val_152 -152val_152 val_152 -152val_152 val_152 -152val_152 val_152 -156val_156 val_156 -158val_158 val_158 -170val_170 val_170 -172val_172 val_172 -172val_172 val_172 -172val_172 val_172 -172val_172 val_172 -174val_174 val_174 -174val_174 val_174 -174val_174 val_174 -174val_174 val_174 -176val_176 val_176 -176val_176 val_176 -176val_176 val_176 -176val_176 val_176 -178val_178 val_178 -190val_190 val_190 -192val_192 val_192 -194val_194 val_194 -196val_196 val_196 -200val_200 val_200 -200val_200 val_200 -200val_200 val_200 -200
svn commit: r1580671 - in /hive/trunk/serde/src: java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java test/org/apache/hadoo
Author: vikram
Date: Mon Mar 24 00:18:55 2014
New Revision: 1580671
URL: http://svn.apache.org/r1580671
Log:
HIVE-6707 : Lazy maps are broken (LazyMap and LazyBinaryMap) (Prasanth J via
Vikram Dixit, reviewed by Gunther)
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java?rev=1580671r1=1580670r2=1580671view=diff
==
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
Mon Mar 24 00:18:55 2014
@@ -67,6 +67,23 @@ public abstract class LazyPrimitiveOI e
return isNull ? 0 : data.hashCode();
}
+ @Override
+ public boolean equals(Object obj) {
+if (!(obj instanceof LazyPrimitive?, ?)) {
+ return false;
+}
+
+if (data == obj) {
+ return true;
+}
+
+if (data == null || obj == null) {
+ return false;
+}
+
+return data.equals(((LazyPrimitive?, ?) obj).getWritableObject());
+ }
+
public void logExceptionMessage(ByteArrayRef bytes, int start, int length,
String dataType) {
try {
if(LOG.isDebugEnabled()) {
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java?rev=1580671r1=1580670r2=1580671view=diff
==
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
Mon Mar 24 00:18:55 2014
@@ -65,4 +65,21 @@ public abstract class LazyBinaryPrimitiv
public int hashCode() {
return data == null ? 0 : data.hashCode();
}
+
+ @Override
+ public boolean equals(Object obj) {
+if (!(obj instanceof LazyBinaryPrimitive?, ?)) {
+ return false;
+}
+
+if (data == obj) {
+ return true;
+}
+
+if (data == null || obj == null) {
+ return false;
+}
+
+return data.equals(((LazyBinaryPrimitive?, ?) obj).getWritableObject());
+ }
}
Modified:
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1580671r1=1580670r2=1580671view=diff
==
---
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
(original)
+++
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Mon Mar 24 00:18:55 2014
@@ -31,6 +31,9 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap;
+import
org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -179,7 +182,7 @@ public class TestLazyArrayMapStruct exte
.getMapValueElement(new Text(8))).getWritableObject());
assertNull(b.getMapValueElement(new Text(-)));
-assertEquals({'2':'d\\tf','2':'d','-1':null,'0':'0','8':'abc'}
+assertEquals({'2':'d\\tf','-1':null,'0':'0','8':'abc'}
.replace('\'', '\'), SerDeUtils.getJSONString(b, oi));
}
@@ -190,6 +193,67 @@ public class TestLazyArrayMapStruct exte
}
/**
+ * Test the LazyMap class.
+ */
+ public void testLazyMapWithDuplicateKeys() throws Throwable {
+try {
+ {
+// Map of Integer to String
+Text nullSequence = new Text(\\N);
+ObjectInspector oi = LazyFactory
+.createLazyObjectInspector(TypeInfoUtils
+.getTypeInfosFromTypeString(mapint,string).get(0),
+new byte[] {(byte) 1, (byte) 2}, 0, nullSequence, false,
+(byte) 0);
+LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+byte[] data = new byte[] {'2', 2, 'd', 'e
svn commit: r1580672 - /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
Author: vikram
Date: Mon Mar 24 00:23:01 2014
New Revision: 1580672
URL: http://svn.apache.org/r1580672
Log:
HIVE-6711 : ORC maps uses getMapSize() from MapOI which is unreliable (Prasanth
J via Vikram Dixit, reviewed by Gunther)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1580672r1=1580671r2=1580672view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Mon
Mar 24 00:23:01 2014
@@ -1483,11 +1483,10 @@ class WriterImpl implements Writer, Memo
super.write(obj);
if (obj != null) {
MapObjectInspector insp = (MapObjectInspector) inspector;
-int len = insp.getMapSize(obj);
-lengths.write(len);
// this sucks, but it will have to do until we can get a better
// accessor in the MapObjectInspector.
Map?, ? valueMap = insp.getMap(obj);
+lengths.write(valueMap.size());
for(Map.Entry?, ? entry: valueMap.entrySet()) {
childrenWriters[0].write(entry.getKey());
childrenWriters[1].write(entry.getValue());
svn commit: r1580797 - in /hive/branches/branch-0.13/ql/src: java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java test/queries/clientpositive/tez_dml.q test/results/clientpositive/tez/tez_dml.q.out
Author: vikram
Date: Mon Mar 24 09:57:42 2014
New Revision: 1580797
URL: http://svn.apache.org/r1580797
Log:
HIVE-6707 : Lazy maps are broken (LazyMap and LazyBinaryMap) (Prasanth J via
Vikram Dixit, reviewed by Gunther)
Modified:
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_dml.q
hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_dml.q.out
Modified:
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1580797r1=1580796r2=1580797view=diff
==
---
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
(original)
+++
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
Mon Mar 24 09:57:42 2014
@@ -308,10 +308,6 @@ public class HiveInputFormatK extends W
try {
ListPath paths = Utilities.getInputPathsTez(job, mrwork);
dirs = paths.toArray(new Path[paths.size()]);
- if (dirs.length == 0) {
-// if we still don't have any files it's time to fail.
-throw new IOException(No input paths specified in job);
- }
} catch (Exception e) {
throw new IOException(Could not create input files, e);
}
@@ -373,11 +369,13 @@ public class HiveInputFormatK extends W
currentInputFormatClass = inputFormatClass;
}
-LOG.info(Generating splits);
-addSplitsForGroup(currentDirs, currentTableScan, newjob,
-getInputFormatFromCache(currentInputFormatClass, job),
-currentInputFormatClass, currentDirs.size()*(numSplits / dirs.length),
-currentTable, result);
+if (dirs.length != 0) {
+ LOG.info(Generating splits);
+ addSplitsForGroup(currentDirs, currentTableScan, newjob,
+ getInputFormatFromCache(currentInputFormatClass, job),
+ currentInputFormatClass, currentDirs.size()*(numSplits /
dirs.length),
+ currentTable, result);
+}
LOG.info(number of splits + result.size());
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS);
Modified: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_dml.q
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_dml.q?rev=1580797r1=1580796r2=1580797view=diff
==
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_dml.q
(original)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/tez_dml.q Mon
Mar 24 09:57:42 2014
@@ -29,6 +29,10 @@ INSERT INTO TABLE odd SELECT key, value
SELECT * FROM even;
SELECT * FROM odd;
+-- create empty table
+CREATE TABLE empty STORED AS orc AS SELECT * FROM tmp_src_part WHERE d = -1000;
+SELECT * FROM empty;
+
-- drop the tables
DROP TABLE even;
DROP TABLE odd;
Modified:
hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_dml.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_dml.q.out?rev=1580797r1=1580796r2=1580797view=diff
==
---
hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_dml.q.out
(original)
+++
hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/tez_dml.q.out
Mon Mar 24 09:57:42 2014
@@ -1499,6 +1499,41 @@ POSTHOOK: Lineage: tmp_src_part PARTITIO
169val_169
403val_403
97 val_97
+PREHOOK: query: -- create empty table
+CREATE TABLE empty STORED AS orc AS SELECT * FROM tmp_src_part WHERE d = -1000
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@tmp_src_part
+POSTHOOK: query: -- create empty table
+CREATE TABLE empty STORED AS orc AS SELECT * FROM tmp_src_part WHERE d = -1000
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@tmp_src_part
+POSTHOOK: Output: default@empty
+POSTHOOK: Lineage: even.c EXPRESSION [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: even.d SIMPLE [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
+POSTHOOK: Lineage: odd.c EXPRESSION [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: odd.d SIMPLE [(src)src.FieldSchema(name:value, type:string,
comment:default), ]
+POSTHOOK: Lineage: tmp_src_part PARTITION(d=1).c SIMPLE
[(tmp_src)tmp_src.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: tmp_src_part PARTITION(d=2).c SIMPLE
[(tmp_src)tmp_src.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: tmp_src_part PARTITION(d=3).c SIMPLE
[(tmp_src)tmp_src.FieldSchema(name:value
svn commit: r1580802 - /hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
Author: vikram
Date: Mon Mar 24 10:19:54 2014
New Revision: 1580802
URL: http://svn.apache.org/r1580802
Log:
HIVE-6711 : ORC maps uses getMapSize() from MapOI which is unreliable (Prasanth
J via Vikram Dixit, reviewed by Gunther)
Modified:
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
Modified:
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1580802r1=1580801r2=1580802view=diff
==
---
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
(original)
+++
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
Mon Mar 24 10:19:54 2014
@@ -1483,11 +1483,10 @@ class WriterImpl implements Writer, Memo
super.write(obj);
if (obj != null) {
MapObjectInspector insp = (MapObjectInspector) inspector;
-int len = insp.getMapSize(obj);
-lengths.write(len);
// this sucks, but it will have to do until we can get a better
// accessor in the MapObjectInspector.
Map?, ? valueMap = insp.getMap(obj);
+lengths.write(valueMap.size());
for(Map.Entry?, ? entry: valueMap.entrySet()) {
childrenWriters[0].write(entry.getKey());
childrenWriters[1].write(entry.getValue());
svn commit: r1582613 [4/4] - in /hive/branches/branch-0.13: common/src/java/org/apache/hadoop/hive/conf/ itests/qtest/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql
Added: hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out?rev=1582613view=auto == --- hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out (added) +++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out Fri Mar 28 05:53:12 2014 @@ -0,0 +1,710 @@ +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab_part +PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin_part +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD + A masked pattern was here +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD + A masked pattern was here +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD + A masked pattern was here +POSTHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD + A masked pattern was here +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD + A masked pattern was here +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part
svn commit: r1582623 [2/4] - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ itests/qtest/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/s
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateOpTraitsProcCtx.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateOpTraitsProcCtx.java?rev=1582623view=auto
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateOpTraitsProcCtx.java
(added)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateOpTraitsProcCtx.java
Fri Mar 28 06:11:07 2014
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.metainfo.annotation;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+
+public class AnnotateOpTraitsProcCtx implements NodeProcessorCtx {
+
+ ParseContext parseContext;
+ HiveConf conf;
+
+ public AnnotateOpTraitsProcCtx(ParseContext parseContext) {
+this.setParseContext(parseContext);
+if(parseContext != null) {
+ this.setConf(parseContext.getConf());
+} else {
+ this.setConf(null);
+}
+ }
+
+ public HiveConf getConf() {
+return conf;
+ }
+
+ public void setConf(HiveConf conf) {
+this.conf = conf;
+ }
+
+ public ParseContext getParseContext() {
+return parseContext;
+ }
+
+ public void setParseContext(ParseContext parseContext) {
+this.parseContext = parseContext;
+ }
+
+}
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java?rev=1582623view=auto
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java
(added)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java
Fri Mar 28 06:11:07 2014
@@ -0,0 +1,78 @@
+package org.apache.hadoop.hive.ql.optimizer.metainfo.annotation;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.exec.DemuxOperator;
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.MuxOperator;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.exec.UnionOperator;
+import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.optimizer.Transform;
+import
org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.OpTraitsRulesProcFactory;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+/*
+ * This class annotates each operator with its traits. The OpTraits class
+ * specifies the traits that are populated for each operator.
+ */
+public class AnnotateWithOpTraits implements Transform {
+
+ @Override
+ public ParseContext transform(ParseContext pctx) throws SemanticException {
+AnnotateOpTraitsProcCtx annotateCtx = new AnnotateOpTraitsProcCtx(pctx);
+
+// create a walker which walks the tree in a DFS manner while maintaining
the
+// operator stack. The dispatcher generates the plan from the operator tree
+MapRule, NodeProcessor opRules = new LinkedHashMapRule,
NodeProcessor();
+
svn commit: r1585810 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/ common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/h
Author: vikram
Date: Tue Apr 8 18:25:42 2014
New Revision: 1585810
URL: http://svn.apache.org/r1585810
Log:
HIVE-6782 : HiveServer2Concurrency issue when running with tez intermittently,
throwing org.apache.tez.dag.api.SessionNotRunning: Application not running
error (Vikram Dixit K, reviewed by Thejas Nair)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1585810r1=1585809r2=1585810view=diff
==
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Tue
Apr 8 18:25:42 2014
@@ -441,6 +441,4 @@ public final class FileUtils {
}
return true;
}
-
-
}
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1585810r1=1585809r2=1585810view=diff
==
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue
Apr 8 18:25:42 2014
@@ -1020,7 +1020,8 @@ public class HiveConf extends Configurat
// Check if a plan contains a Cross Product.
// If there is one, output a warning to the Session's console.
HIVE_CHECK_CROSS_PRODUCT(hive.exec.check.crossproducts, true),
-
+
HIVE_LOCALIZE_RESOURCE_WAIT_INTERVAL(hive.localize.resource.wait.interval,
5000L), // in ms
+
HIVE_LOCALIZE_RESOURCE_NUM_WAIT_ATTEMPTS(hive.localize.resource.num.wait.attempts,
5),
;
public final String varname;
Modified: hive/trunk/conf/hive-default.xml.template
URL:
http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1585810r1=1585809r2=1585810view=diff
==
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Tue Apr 8 18:25:42 2014
@@ -2604,4 +2604,20 @@
/description
/property
+property
+ namehive.localize.resource.wait.interval/name
+ value5000/value
+ description
+Time in milliseconds to wait for another thread to localize the same
resource for hive-tez.
+ /description
+/property
+
+property
+ namehive.localize.resource.num.wait.attempts/name
+ value5/value
+ description
+The number of attempts waiting for localizing a resource in hive-tez.
+ /description
+/property
+
/configuration
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1585810r1=1585809r2=1585810view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java Tue Apr 8
18:25:42 2014
@@ -30,13 +30,16 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.TaskRunner;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import java.io.DataInput;
@@ -51,6 +54,8 @@ import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
+import javax.security.auth.login.LoginException;
+
/**
* Context for Semantic Analyzers. Usage
svn commit: r1586455 - in /hive/branches/branch-0.13: common/src/main/resources/hive-log4j.properties ql/src/main/resources/hive-exec-log4j.properties
Author: vikram
Date: Thu Apr 10 20:02:12 2014
New Revision: 1586455
URL: http://svn.apache.org/r1586455
Log:
HIVE-6876 : Logging information should include thread id (Vikram Dixit K,
reviewed by Jason Dere)
Modified:
hive/branches/branch-0.13/common/src/main/resources/hive-log4j.properties
hive/branches/branch-0.13/ql/src/main/resources/hive-exec-log4j.properties
Modified:
hive/branches/branch-0.13/common/src/main/resources/hive-log4j.properties
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/common/src/main/resources/hive-log4j.properties?rev=1586455r1=1586454r2=1586455view=diff
==
--- hive/branches/branch-0.13/common/src/main/resources/hive-log4j.properties
(original)
+++ hive/branches/branch-0.13/common/src/main/resources/hive-log4j.properties
Thu Apr 10 20:02:12 2014
@@ -48,7 +48,7 @@ log4j.appender.DRFA.layout=org.apache.lo
# Pattern format: Date LogLevel LoggerName LogMessage
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2}
(%F:%M(%L)) - %m%n
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2}
(%F:%M(%L)) - %m%n
#
@@ -59,7 +59,7 @@ log4j.appender.DRFA.layout.ConversionPat
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p
%c{2}: %m%n
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p
%c{2}: %m%n
log4j.appender.console.encoding=UTF-8
#custom logging levels
Modified:
hive/branches/branch-0.13/ql/src/main/resources/hive-exec-log4j.properties
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/main/resources/hive-exec-log4j.properties?rev=1586455r1=1586454r2=1586455view=diff
==
--- hive/branches/branch-0.13/ql/src/main/resources/hive-exec-log4j.properties
(original)
+++ hive/branches/branch-0.13/ql/src/main/resources/hive-exec-log4j.properties
Thu Apr 10 20:02:12 2014
@@ -38,7 +38,7 @@ log4j.appender.FA.layout=org.apache.log4
# Pattern format: Date LogLevel LoggerName LogMessage
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
# Debugging Pattern format
-log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L))
- %m%n
+log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2}
(%F:%M(%L)) - %m%n
#
@@ -49,7 +49,7 @@ log4j.appender.FA.layout.ConversionPatte
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p
%c{2}: %m%n
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p
%c{2}: %m%n
#custom logging levels
#log4j.logger.xxx=DEBUG
svn commit: r1592645 - in /hive/trunk: itests/qtest/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/tez/
Author: vikram Date: Tue May 6 00:17:54 2014 New Revision: 1592645 URL: http://svn.apache.org/r1592645 Log: HIVE-6826 : Hive-tez has issues when different partitions work off of different input types (Vikram Dixit K, reviewed by Gunther Hagleitner) Added: hive/trunk/ql/src/test/queries/clientpositive/tez_schema_evolution.q hive/trunk/ql/src/test/results/clientpositive/tez/tez_schema_evolution.q.out Modified: hive/trunk/itests/qtest/pom.xml hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Modified: hive/trunk/itests/qtest/pom.xml URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest/pom.xml?rev=1592645r1=1592644r2=1592645view=diff == --- hive/trunk/itests/qtest/pom.xml (original) +++ hive/trunk/itests/qtest/pom.xml Tue May 6 00:17:54 2014 @@ -38,7 +38,7 @@ execute.beeline.testsfalse/execute.beeline.tests minimr.query.filesstats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q/minimr.query.files minimr.query.negative.filescluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q/minimr.query.negative.files - minitez.query.filestez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q/minitez.query.files + minitez.query.filestez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q/minitez.query.files minitez.query.files.sharedcross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q/minitez.query.files.shared beeline.positive.excludeadd_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rena me.q,exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q
svn commit: r1597845 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/tez/ test/org/apache/hadoop/hive/ql/exec/tez/
Author: vikram
Date: Tue May 27 18:27:51 2014
New Revision: 1597845
URL: http://svn.apache.org/r1597845
Log:
HIVE-7043: When using the tez session pool via hive, once sessions time out,
all queries go to the default queue: (Vikram Dixit, reviewed by Gunther
Hagleitner)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java?rev=1597845r1=1597844r2=1597845view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
Tue May 27 18:27:51 2014
@@ -78,7 +78,7 @@ public class TezJobMonitor {
try {
for (TezSessionState s: TezSessionState.getOpenSessions()) {
System.err.println(Shutting down tez session.);
-s.close(false);
+TezSessionPoolManager.getInstance().close(s);
}
} catch (Exception e) {
// ignore
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java?rev=1597845r1=1597844r2=1597845view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
Tue May 27 18:27:51 2014
@@ -262,9 +262,19 @@ public class TezSessionPoolManager {
}
if (session != null) {
- session.close(false);
+ close(session);
}
return getSession(conf, doOpen, forceCreate);
}
+
+ public void closeAndOpen(TezSessionState sessionState, HiveConf conf)
+ throws Exception {
+HiveConf sessionConf = sessionState.getConf();
+if (sessionConf.get(tez.queue.name) != null) {
+ conf.set(tez.queue.name, sessionConf.get(tez.queue.name));
+}
+close(sessionState);
+sessionState.open(conf);
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1597845r1=1597844r2=1597845view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Tue
May 27 18:27:51 2014
@@ -18,8 +18,6 @@
package org.apache.hadoop.hive.ql.exec.tez;
-import java.io.IOException;
-import java.net.URISyntaxException;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -28,8 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import javax.security.auth.login.LoginException;
-
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -39,7 +35,6 @@ import org.apache.hadoop.hive.ql.exec.Op
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.log.PerfLogger;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.TezEdgeProperty;
import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType;
@@ -57,7 +52,6 @@ import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.GroupInputEdge;
import org.apache.tez.dag.api.SessionNotRunning;
-import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.VertexGroup;
import org.apache.tez.dag.api.client.DAGClient;
@@ -297,8 +291,7 @@ public class TezTask extends TaskTezWor
DAGClient submit(JobConf conf, DAG dag, Path scratchDir,
LocalResource appJarLr, TezSessionState sessionState)
- throws IOException, TezException, InterruptedException,
- LoginException, URISyntaxException, HiveException {
+ throws Exception {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG);
DAGClient dagClient = null;
@@ -310,11 +303,7 @@ public class TezTask extends TaskTezWor
console.printInfo(Tez session was closed. Reopening...);
// close the old one, but keep the tmp files around
svn commit: r1614949 - in /hive/branches/tez: itests/qtest/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ ql/src/java/org/apache/hadoop/hive/ql/exec/persis
Author: vikram Date: Thu Jul 31 18:27:18 2014 New Revision: 1614949 URL: http://svn.apache.org/r1614949 Log: HIVE-7096: Support grouped splits in Tez partitioned broadcast join (Vikram Dixit, reviewed by Gunther Hagleitner) Modified: hive/branches/tez/itests/qtest/testconfiguration.properties hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/DefaultHivePartitioner.java Modified: hive/branches/tez/itests/qtest/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/qtest/testconfiguration.properties?rev=1614949r1=1614948r2=1614949view=diff == --- hive/branches/tez/itests/qtest/testconfiguration.properties (original) +++ hive/branches/tez/itests/qtest/testconfiguration.properties Thu Jul 31 18:27:18 2014 @@ -1,5 +1,5 @@ minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q,empty_dir_in_table.q,temp_table_external.q minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q -minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q +minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q,tez_bmj_schema_evolution.q minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t able.q,vectorized_ptf.q,optimize_nullscan.q beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q
svn commit: r1614973 - in /hive/branches/tez: itests/qtest/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ ql/src/java/org/apache/hadoop/hive/ql/exec/persis
Author: vikram Date: Thu Jul 31 20:00:57 2014 New Revision: 1614973 URL: http://svn.apache.org/r1614973 Log: HIVE-7096: Support grouped splits in Tez partitioned broadcast join (Vikram Dixit, reviewed by Gunther Hagleitner) Added: hive/branches/tez/ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q hive/branches/tez/ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out Modified: hive/branches/tez/itests/qtest/testconfiguration.properties hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/DefaultHivePartitioner.java Modified: hive/branches/tez/itests/qtest/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/qtest/testconfiguration.properties?rev=1614973r1=1614972r2=1614973view=diff == --- hive/branches/tez/itests/qtest/testconfiguration.properties (original) +++ hive/branches/tez/itests/qtest/testconfiguration.properties Thu Jul 31 20:00:57 2014 @@ -1,5 +1,5 @@ minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q,empty_dir_in_table.q,temp_table_external.q minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q -minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q,tez_bmj_schema_evolution.q +minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t able.q,vectorized_ptf.q,optimize_nullscan.q beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q
svn commit: r1615007 - in /hive/branches/tez: itests/qtest/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ ql/src/java/org/apache/hadoop/hive/ql/exec/persis
Author: vikram Date: Thu Jul 31 20:54:30 2014 New Revision: 1615007 URL: http://svn.apache.org/r1615007 Log: HIVE-7096: Support grouped splits in Tez partitioned broadcast join (Vikram Dixit, reviewed by Gunther Hagleitner) Added: hive/branches/tez/ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q hive/branches/tez/ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out Modified: hive/branches/tez/itests/qtest/testconfiguration.properties hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/DefaultHivePartitioner.java Modified: hive/branches/tez/itests/qtest/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/qtest/testconfiguration.properties?rev=1615007r1=1615006r2=1615007view=diff == --- hive/branches/tez/itests/qtest/testconfiguration.properties (original) +++ hive/branches/tez/itests/qtest/testconfiguration.properties Thu Jul 31 20:54:30 2014 @@ -1,5 +1,5 @@ minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q,empty_dir_in_table.q,temp_table_external.q minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q -minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q +minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q,tez_bmj_schema_evolution.q minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t able.q,vectorized_ptf.q,optimize_nullscan.q beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q
svn commit: r1556059 - in /hive/site: author/src/documentation/content/xdocs/credits.xml publish/credits.html publish/credits.pdf
Author: vikram Date: Mon Jan 6 23:05:25 2014 New Revision: 1556059 URL: http://svn.apache.org/r1556059 Log: HIVE-6153: Add Vikram to list of committers. Add Gunther to PMC list. Sort names. Modified: hive/site/author/src/documentation/content/xdocs/credits.xml hive/site/publish/credits.html hive/site/publish/credits.pdf Modified: hive/site/author/src/documentation/content/xdocs/credits.xml URL: http://svn.apache.org/viewvc/hive/site/author/src/documentation/content/xdocs/credits.xml?rev=1556059r1=1556058r2=1556059view=diff == --- hive/site/author/src/documentation/content/xdocs/credits.xml (original) +++ hive/site/author/src/documentation/content/xdocs/credits.xml Mon Jan 6 23:05:25 2014 @@ -51,6 +51,12 @@ /tr tr +tdgunther/td !-- username -- +tdGunther Hagleitner/td !-- name -- +tdHortonworks/td !-- organization -- + /tr + + tr tdhashutosh/td !-- username -- tdAshutosh Chauhan/td !-- name -- tdHortonWorks/td !-- organization -- @@ -105,6 +111,7 @@ tda href=http://hortonworks.com/;Hortonworks/a/td !-- organization -- td/td !-- roles -- /tr + /table /section @@ -189,7 +196,7 @@ tdHarish Butani/td !-- name -- tda href=http://hortonworks.com/;Hortonworks/a/td !-- organization -- /tr - + tr tdsdong/td !-- username -- tdSiying Dong/td !-- name -- @@ -197,9 +204,9 @@ /tr tr -tdyhuai/td !-- username -- -tda href=http://www.cse.ohio-state.edu/~huai/;Yin Huai/a/td !-- name -- -tda href=http://www.osu.edu/;The Ohio State University/a/td !-- organization -- +tdvikram/td !-- username -- +tdVikram Dixit/td !-- name -- +tda href=http://www.hortonworks.com/;Hortonworks/a/td !-- organization -- /tr tr @@ -208,6 +215,12 @@ tdCloudera/td !-- organization -- /tr + tr +tdyhuai/td !-- username -- +tda href=http://www.cse.ohio-state.edu/~huai/;Yin Huai/a/td !-- name -- +tda href=http://www.osu.edu/;The Ohio State University/a/td !-- organization -- + /tr + /table /section Modified: hive/site/publish/credits.html URL: http://svn.apache.org/viewvc/hive/site/publish/credits.html?rev=1556059r1=1556058r2=1556059view=diff == --- hive/site/publish/credits.html (original) +++ hive/site/publish/credits.html Mon Jan 6 23:05:25 2014 @@ -268,6 +268,15 @@ document.write(Last Published: + docu tr +td colspan=1 rowspan=1gunther/td !-- username -- +td colspan=1 rowspan=1Gunther Hagleitner/td !-- name -- +td colspan=1 rowspan=1Hortonworks/td !-- organization -- + +/tr + + +tr + td colspan=1 rowspan=1hashutosh/td !-- username -- td colspan=1 rowspan=1Ashutosh Chauhan/td !-- name -- td colspan=1 rowspan=1HortonWorks/td !-- organization -- @@ -344,13 +353,14 @@ document.write(Last Published: + docu td colspan=1 rowspan=1/td !-- roles -- /tr + /table /div -a name=N10199/aa name=Emeritus+Hive+PMC+Members/a +a name=N101B1/aa name=Emeritus+Hive+PMC+Members/a h2 class=h3Emeritus Hive PMC Members/h2 div class=section ul @@ -371,7 +381,7 @@ document.write(Last Published: + docu /div -a name=N101B5/aa name=Hive+Committers/a +a name=N101CD/aa name=Hive+Committers/a h2 class=h3Hive Committers/h2 div class=section table class=ForrestTable cellspacing=1 cellpadding=4 @@ -473,7 +483,7 @@ document.write(Last Published: + docu td colspan=1 rowspan=1a href=http://hortonworks.com/;Hortonworks/a/td !-- organization -- /tr - + tr @@ -486,9 +496,9 @@ document.write(Last Published: + docu tr -td colspan=1 rowspan=1yhuai/td !-- username -- -td colspan=1 rowspan=1a href=http://www.cse.ohio-state.edu/~huai/;Yin Huai/a/td !-- name -- -td colspan=1 rowspan=1a href=http://www.osu.edu/;The Ohio State University/a/td !-- organization -- +td colspan=1 rowspan=1vikram/td !-- username -- +td colspan=1 rowspan=1Vikram Dixit/td !-- name -- +td colspan=1 rowspan=1a href=http://www.hortonworks.com/;Hortonworks/a/td !-- organization -- /tr @@ -501,12 +511,21 @@ document.write(Last Published: + docu /tr + +tr + +td colspan=1 rowspan=1yhuai/td !-- username -- +td colspan=1 rowspan=1a href=http://www.cse.ohio-state.edu/~huai/;Yin Huai/a/td !-- name -- +td colspan=1 rowspan=1a href=http://www.osu.edu/;The Ohio State University/a/td !-- organization -- + +/tr + /table /div -a name=N1030F/aa name=HCatalog+Committers/a +a name=N10341/aa name=HCatalog+Committers/a h2 class=h3HCatalog Committers/h2 div class
svn commit: r1569619 - /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
Author: vikram
Date: Wed Feb 19 02:50:09 2014
New Revision: 1569619
URL: http://svn.apache.org/r1569619
Log:
HIVE-6326: Split generation in ORC may generate wrong split boundaries because
of unaccounted padded bytes (Prasanth J via Vikram Dixit K)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1569619r1=1569618r2=1569619view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
Wed Feb 19 02:50:09 2014
@@ -729,7 +729,10 @@ public class OrcInputFormat implements
idx++;
// eliminate stripes that doesn't satisfy the predicate condition
- if (sarg != null !isStripeSatisfyPredicate(stripeStats.get(idx),
sarg, filterColumns)) {
+ if (sarg != null
+ stripeStats != null
+ idx stripeStats.size()
+ !isStripeSatisfyPredicate(stripeStats.get(idx), sarg,
filterColumns)) {
// if a stripe doesn't satisfy predicate condition then skip it
if (LOG.isDebugEnabled()) {
@@ -757,7 +760,7 @@ public class OrcInputFormat implements
currentOffset = stripe.getOffset();
currentLength = stripe.getLength();
} else {
-currentLength += stripe.getLength();
+currentLength = (stripe.getOffset() + stripe.getLength()) -
currentOffset;
}
if (currentLength = context.maxSize) {
createSplit(currentOffset, currentLength, fileMetaInfo);
svn commit: r1574640 - in /hive/branches/branch-0.13: common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/java/org/apache/hadoop/hive/ql/session/
Author: vikram Date: Wed Mar 5 19:55:24 2014 New Revision: 1574640 URL: http://svn.apache.org/r1574640 Log: HIVE-6325: Enable using multiple concurrent sessions in tez (Vikram Dixit, reviewed by Gunther Hagleitner) Added: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java Modified: hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java hive/branches/branch-0.13/conf/hive-default.xml.template hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java hive/branches/branch-0.13/service/src/java/org/apache/hive/service/server/HiveServer2.java Modified: hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1574640r1=1574639r2=1574640view=diff == --- hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original) +++ hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Mar 5 19:55:24 2014 @@ -956,6 +956,11 @@ public class HiveConf extends Configurat HIVECOUNTERGROUP(hive.counters.group.name, HIVE), +HIVE_SERVER2_TEZ_DEFAULT_QUEUES(hive.server2.tez.default.queues, ), + HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE(hive.server2.tez.sessions.per.default.queue, 1), + HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS(hive.server2.tez.initialize.default.sessions, +false), + // none, column // none is the default(past) behavior. Implies only alphaNumeric and underscore are valid characters in identifiers. // column: implies column names can contain any character. Modified: hive/branches/branch-0.13/conf/hive-default.xml.template URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/conf/hive-default.xml.template?rev=1574640r1=1574639r2=1574640view=diff == --- hive/branches/branch-0.13/conf/hive-default.xml.template (original) +++ hive/branches/branch-0.13/conf/hive-default.xml.template Wed Mar 5 19:55:24 2014 @@ -2406,4 +2406,34 @@ descriptionBy default tez will use the java opts from map tasks. This can be used to overwrite./description /property +property + namehive.server2.tez.default.queues/name + value/value + description +A list of comma separated values corresponding to yarn queues of the same name. +When hive server 2 is launched in tez mode, this configuration needs to be set +for multiple tez sessions to run in parallel on the cluster. + /description +/property + +property + namehive.server2.tez.sessions.per.default.queue/name + value1/value + description +A positive integer that determines the number of tez sessions that should be +launched on each of the queues specified by hive.server2.tez.default.queues. +Determines the parallelism on each queue. + /description +/property + +property + namehive.server2.tez.initialize.default.sessions/name + valuefalse/value + description +This flag is used in hive server 2 to enable a user to use hive server 2 without +turning on tez for hive server 2. The user could potentially want to run queries +over tez without the pool of sessions. + /description +/property + /configuration Added: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java?rev=1574640view=auto == --- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java (added) +++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java Wed Mar 5 19:55:24 2014 @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless
svn commit: r1576458 - /hive/trunk/ql/pom.xml
Author: vikram
Date: Tue Mar 11 18:28:57 2014
New Revision: 1576458
URL: http://svn.apache.org/r1576458
Log:
HIVE-6563: hdfs jar being pulled in when creating a hadoop-2 based hive tar
ball (Vikram Dixit, reviewed by Harish Butani)
Modified:
hive/trunk/ql/pom.xml
Modified: hive/trunk/ql/pom.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/pom.xml?rev=1576458r1=1576457r2=1576458view=diff
==
--- hive/trunk/ql/pom.xml (original)
+++ hive/trunk/ql/pom.xml Tue Mar 11 18:28:57 2014
@@ -354,6 +354,7 @@
groupIdorg.apache.hadoop/groupId
artifactIdhadoop-hdfs/artifactId
version${hadoop-23.version}/version
+ optionaltrue/optional
/dependency
dependency
groupIdorg.apache.hadoop/groupId
svn commit: r1648620 - /hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Author: vikram
Date: Tue Dec 30 22:38:28 2014
New Revision: 1648620
URL: http://svn.apache.org/r1648620
Log:
HIVE-9205: Change default tez install directory to use /tmp instead of /user
and create the directory if it does not exist (Vikram Dixit K, reviewed by
Prasanth J and Gunther Hagleitner)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1648620r1=1648619r2=1648620view=diff
==
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Tue Dec 30 22:38:28 2014
@@ -771,9 +771,14 @@ public class DagUtils {
String hdfsDirPathStr = jarPathStr;
Path hdfsDirPath = new Path(hdfsDirPathStr);
-FileStatus fstatus = fs.getFileStatus(hdfsDirPath);
-if (!fstatus.isDir()) {
- throw new
IOException(ErrorMsg.INVALID_DIR.format(hdfsDirPath.toString()));
+try {
+ FileStatus fstatus = fs.getFileStatus(hdfsDirPath);
+ if (!fstatus.isDir()) {
+throw new
IOException(ErrorMsg.INVALID_DIR.format(hdfsDirPath.toString()));
+ }
+} catch (FileNotFoundException e) {
+ // directory does not exist, create it
+ fs.mkdirs(hdfsDirPath);
}
Path retPath = new Path(hdfsDirPath.toString() + /.hiveJars);
svn commit: r1654344 - in /hive/branches/branch-1.0/ql/src: java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java test/results/clientpositive/tez/vectorized_parquet.q.out test/result
Author: vikram
Date: Fri Jan 23 19:39:56 2015
New Revision: 1654344
URL: http://svn.apache.org/r1654344
Log:
HIVE-9235: Turn off Parquet Vectorization until all data types work: DECIMAL,
DATE, TIMESTAMP, CHAR, and VARCHAR (Matt McCline via Vikram Dixit K)
Modified:
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
hive/branches/branch-1.0/ql/src/test/results/clientpositive/vectorized_parquet.q.out
Modified:
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java?rev=1654344r1=1654343r2=1654344view=diff
==
---
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
(original)
+++
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
Fri Jan 23 19:39:56 2015
@@ -31,9 +31,10 @@ import parquet.hadoop.ParquetInputFormat
*
* A Parquet InputFormat for Hive (with the deprecated package mapred)
*
+ * NOTE: With HIVE-9235 we removed implements VectorizedParquetInputFormat
since all data types
+ * are not currently supported. Removing the interface turns off
vectorization.
*/
-public class MapredParquetInputFormat extends FileInputFormatVoid,
ArrayWritable
-implements VectorizedInputFormatInterface {
+public class MapredParquetInputFormat extends FileInputFormatVoid,
ArrayWritable {
private static final Log LOG =
LogFactory.getLog(MapredParquetInputFormat.class);
Modified:
hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out?rev=1654344r1=1654343r2=1654344view=diff
==
---
hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
(original)
+++
hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
Fri Jan 23 19:39:56 2015
@@ -149,7 +149,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic
stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type:
smallint), _col3 (type: bigint), _col4 (type:
structcount:bigint,sum:double,input:float), _col5 (type:
structcount:bigint,sum:double,variance:double)
-Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Group By Operator
Modified:
hive/branches/branch-1.0/ql/src/test/results/clientpositive/vectorized_parquet.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/test/results/clientpositive/vectorized_parquet.q.out?rev=1654344r1=1654343r2=1654344view=diff
==
---
hive/branches/branch-1.0/ql/src/test/results/clientpositive/vectorized_parquet.q.out
(original)
+++
hive/branches/branch-1.0/ql/src/test/results/clientpositive/vectorized_parquet.q.out
Fri Jan 23 19:39:56 2015
@@ -84,7 +84,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -161,7 +160,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic stats:
COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type:
smallint), _col3 (type: bigint), _col4 (type:
structcount:bigint,sum:double,input:float), _col5 (type:
structcount:bigint,sum:double,variance:double)
- Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), min(VALUE._col1),
count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4)
svn commit: r1654346 - in /hive/branches/branch-0.14/ql/src: java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java test/results/clientpositive/tez/vectorized_parquet.q.out test/resul
Author: vikram
Date: Fri Jan 23 19:43:53 2015
New Revision: 1654346
URL: http://svn.apache.org/r1654346
Log:
HIVE-9235: Turn off Parquet Vectorization until all data types work: DECIMAL,
DATE, TIMESTAMP, CHAR, and VARCHAR (Matt McCline via Vikram Dixit K)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vectorized_parquet.q.out
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java?rev=1654346r1=1654345r2=1654346view=diff
==
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
Fri Jan 23 19:43:53 2015
@@ -31,9 +31,10 @@ import parquet.hadoop.ParquetInputFormat
*
* A Parquet InputFormat for Hive (with the deprecated package mapred)
*
+ * NOTE: With HIVE-9235 we removed implements VectorizedParquetInputFormat
since all data types
+ * are not currently supported. Removing the interface turns off
vectorization.
*/
-public class MapredParquetInputFormat extends FileInputFormatVoid,
ArrayWritable
-implements VectorizedInputFormatInterface {
+public class MapredParquetInputFormat extends FileInputFormatVoid,
ArrayWritable {
private static final Log LOG =
LogFactory.getLog(MapredParquetInputFormat.class);
Modified:
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out?rev=1654346r1=1654345r2=1654346view=diff
==
---
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
(original)
+++
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
Fri Jan 23 19:43:53 2015
@@ -149,7 +149,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic
stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type:
smallint), _col3 (type: bigint), _col4 (type:
structcount:bigint,sum:double,input:float), _col5 (type:
structcount:bigint,sum:double,variance:double)
-Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Group By Operator
Modified:
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vectorized_parquet.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/vectorized_parquet.q.out?rev=1654346r1=1654345r2=1654346view=diff
==
---
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vectorized_parquet.q.out
(original)
+++
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vectorized_parquet.q.out
Fri Jan 23 19:43:53 2015
@@ -84,7 +84,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -161,7 +160,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic stats:
COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type:
smallint), _col3 (type: bigint), _col4 (type:
structcount:bigint,sum:double,input:float), _col5 (type:
structcount:bigint,sum:double,variance:double)
- Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), min(VALUE._col1),
count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4)
svn commit: r1654341 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java test/results/clientpositive/tez/vectorized_parquet.q.out test/results/clientpositi
Author: vikram
Date: Fri Jan 23 19:37:41 2015
New Revision: 1654341
URL: http://svn.apache.org/r1654341
Log:
HIVE-9235: Turn off Parquet Vectorization until all data types work: DECIMAL,
DATE, TIMESTAMP, CHAR, and VARCHAR (Matt McCline via Vikram Dixit K)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorized_parquet.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java?rev=1654341r1=1654340r2=1654341view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
Fri Jan 23 19:37:41 2015
@@ -31,9 +31,10 @@ import parquet.hadoop.ParquetInputFormat
*
* A Parquet InputFormat for Hive (with the deprecated package mapred)
*
+ * NOTE: With HIVE-9235 we removed implements VectorizedParquetInputFormat
since all data types
+ * are not currently supported. Removing the interface turns off
vectorization.
*/
-public class MapredParquetInputFormat extends FileInputFormatVoid,
ArrayWritable
-implements VectorizedInputFormatInterface {
+public class MapredParquetInputFormat extends FileInputFormatVoid,
ArrayWritable {
private static final Log LOG =
LogFactory.getLog(MapredParquetInputFormat.class);
Modified:
hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out?rev=1654341r1=1654340r2=1654341view=diff
==
--- hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
(original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_parquet.q.out
Fri Jan 23 19:37:41 2015
@@ -149,7 +149,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic
stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type:
smallint), _col3 (type: bigint), _col4 (type:
structcount:bigint,sum:double,input:float), _col5 (type:
structcount:bigint,sum:double,variance:double)
-Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Group By Operator
Modified: hive/trunk/ql/src/test/results/clientpositive/vectorized_parquet.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vectorized_parquet.q.out?rev=1654341r1=1654340r2=1654341view=diff
==
--- hive/trunk/ql/src/test/results/clientpositive/vectorized_parquet.q.out
(original)
+++ hive/trunk/ql/src/test/results/clientpositive/vectorized_parquet.q.out Fri
Jan 23 19:37:41 2015
@@ -84,7 +84,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -161,7 +160,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic stats:
COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type:
smallint), _col3 (type: bigint), _col4 (type:
structcount:bigint,sum:double,input:float), _col5 (type:
structcount:bigint,sum:double,variance:double)
- Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), min(VALUE._col1),
count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4)
svn commit: r939014 - in /websites/production/hive/content: ./ javadocs/
Author: vikram Date: Wed Feb 4 21:44:58 2015 New Revision: 939014 Log: Hive 1.0.0 release. Added: websites/production/hive/content/ - copied from r939013, websites/staging/hive/trunk/content/ websites/production/hive/content/javadocs/ - copied from r939013, websites/production/hive/content/javadocs/
svn commit: r1657402 - in /hive/cms/trunk/content: downloads.mdtext javadoc.mdtext
Author: vikram Date: Wed Feb 4 21:26:00 2015 New Revision: 1657402 URL: http://svn.apache.org/r1657402 Log: Hive 1.0.0 release Modified: hive/cms/trunk/content/downloads.mdtext hive/cms/trunk/content/javadoc.mdtext Modified: hive/cms/trunk/content/downloads.mdtext URL: http://svn.apache.org/viewvc/hive/cms/trunk/content/downloads.mdtext?rev=1657402r1=1657401r2=1657402view=diff == --- hive/cms/trunk/content/downloads.mdtext (original) +++ hive/cms/trunk/content/downloads.mdtext Wed Feb 4 21:26:00 2015 @@ -12,6 +12,11 @@ directory. ## News +### 4 February 2015: release 1.0.0 available +This release works with Hadoop 1.x.y, 2.x.y + +You can look at the complete [JIRA change log for this release][HIVE_1_0_0_CL]. + ### 12 November, 2014: release 0.14.0 available This release works with Hadoop 1.x.y, 2.x.y @@ -46,6 +51,7 @@ This release works with Hadoop 0.20.x, You can look at the complete [JIRA change log for this release][HIVE_10_CL]. [HIVE_DL]: http://www.apache.org/dyn/closer.cgi/hive/ +[HIVE_1_0_0_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12329278styleName=TextprojectId=12310843 [HIVE_14_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12326450styleName=TextprojectId=12310843 [HIVE_13_1_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12326829styleName=TextprojectId=12310843 [HIVE_13_CL]: https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12324986styleName=TextprojectId=12310843 Modified: hive/cms/trunk/content/javadoc.mdtext URL: http://svn.apache.org/viewvc/hive/cms/trunk/content/javadoc.mdtext?rev=1657402r1=1657401r2=1657402view=diff == --- hive/cms/trunk/content/javadoc.mdtext (original) +++ hive/cms/trunk/content/javadoc.mdtext Wed Feb 4 21:26:00 2015 @@ -1,5 +1,6 @@ Title: Javadoc + * [Hive 1.0.0 Javadocs][r1.0.0] * [Hive 0.13.1 Javadocs][r0.13.1] * [Hive 0.12.0 Javadocs][r0.12.0] * [Hive 0.11.0 Javadocs][r0.11.0] @@ -8,6 +9,7 @@ Title: Javadoc javadoc and sources jars for use in an IDE are also available via [Nexus][] +[r1.0.0]: /javadocs/r1.0.0/api/index.html [r0.13.1]: /javadocs/r0.13.1/api/index.html [r0.12.0]: /javadocs/r0.12.0/api/index.html [r0.11.0]: /javadocs/r0.11.0/api/index.html
svn commit: r7934 - in /release/hive: ./ hive-1.0.0/
Author: vikram Date: Tue Feb 3 22:50:53 2015 New Revision: 7934 Log: Hive 1.0.0 Release Added: release/hive/hive-1.0.0/ release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz (with props) release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz.asc (with props) release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz.md5 release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz (with props) release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz.asc (with props) release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz.md5 release/hive/hive-1.0.0/stable (with props) Modified: release/hive/stable Added: release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz == Binary file - no diff available. Propchange: release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz -- svn:mime-type = application/x-gzip Added: release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz.asc == Binary file - no diff available. Propchange: release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz.md5 == --- release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz.md5 (added) +++ release/hive/hive-1.0.0/apache-hive-1.0.0-bin.tar.gz.md5 Tue Feb 3 22:50:53 2015 @@ -0,0 +1 @@ +MD5 (./packaging/target/apache-hive-1.0.0-bin.tar.gz) = 16a79ccfecf299202b856039562b88ec Added: release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz == Binary file - no diff available. Propchange: release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz -- svn:mime-type = application/x-gzip Added: release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz.asc == Binary file - no diff available. Propchange: release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz.md5 == --- release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz.md5 (added) +++ release/hive/hive-1.0.0/apache-hive-1.0.0-src.tar.gz.md5 Tue Feb 3 22:50:53 2015 @@ -0,0 +1 @@ +MD5 (./packaging/target/apache-hive-1.0.0-src.tar.gz) = 7fb8368a87203bc24f58a3db1cc93497 Added: release/hive/hive-1.0.0/stable == --- release/hive/hive-1.0.0/stable (added) +++ release/hive/hive-1.0.0/stable Tue Feb 3 22:50:53 2015 @@ -0,0 +1 @@ +link stable \ No newline at end of file Propchange: release/hive/hive-1.0.0/stable -- svn:special = * Modified: release/hive/stable == --- release/hive/stable (original) +++ release/hive/stable Tue Feb 3 22:50:53 2015 @@ -1 +1 @@ -link hive-0.14.0 \ No newline at end of file +link hive-1.0.0/ \ No newline at end of file
svn commit: r7935 - /release/hive/hive-1.0.0/stable
Author: vikram Date: Tue Feb 3 23:11:02 2015 New Revision: 7935 Log: Remove unneeded link Removed: release/hive/hive-1.0.0/stable
svn commit: r1656909 - in /hive/tags: release-1.0.0-rc2/ release-1.0.0/
Author: vikram Date: Tue Feb 3 20:35:46 2015 New Revision: 1656909 URL: http://svn.apache.org/r1656909 Log: Hive 1.0.0 release Added: hive/tags/release-1.0.0/ (props changed) - copied from r1656908, hive/tags/release-1.0.0-rc2/ Removed: hive/tags/release-1.0.0-rc2/ Propchange: hive/tags/release-1.0.0/ -- reviewboard:url = https://reviews.apache.org Propchange: hive/tags/release-1.0.0/ -- --- svn:ignore (added) +++ svn:ignore Tue Feb 3 20:35:46 2015 @@ -0,0 +1,11 @@ +datanucleus.log +eclipse-build +target +.arc_jira_lib +.classpath* +.externalToolBuilders +.project +.settings +*.launch +*~ +metastore_db Propchange: hive/tags/release-1.0.0/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Tue Feb 3 20:35:46 2015 @@ -0,0 +1,5 @@ +/hive/branches/branch-0.11:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184 +/hive/branches/cbo:1605012-1627125 +/hive/branches/tez:1494760-1622766 +/hive/branches/vectorization:1466908-1527856 +/hive/trunk:1627830,1629752,1630512,1635373,1637170,1639249,1655460
svn commit: r1653184 - in /hive/branches/branch-0.14: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/tez
Author: vikram
Date: Tue Jan 20 02:41:40 2015
New Revision: 1653184
URL: http://svn.apache.org/r1653184
Log:
HIVE-9141: HiveOnTez: mix of union all, distinct, group by generates error
(Navis Ryu via Vikram Dixit K)
Added:
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/tez_union2.q
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/tez_union2.q.out
Modified:
hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
Modified:
hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties?rev=1653184r1=1653183r2=1653184view=diff
==
---
hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties
(original)
+++
hive/branches/branch-0.14/itests/src/test/resources/testconfiguration.properties
Tue Jan 20 02:41:40 2015
@@ -277,6 +277,7 @@ minitez.query.files=bucket_map_join_tez1
tez_joins_explain.q,\
tez_schema_evolution.q,\
tez_union.q,\
+ tez_union2.q,\
tez_union_decimal.q,\
tez_union_group_by.q,\
tez_smb_main.q,\
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java?rev=1653184r1=1653183r2=1653184view=diff
==
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
Tue Jan 20 02:41:40 2015
@@ -259,6 +259,16 @@ public class GenTezWork implements NodeP
context.currentMapJoinOperators.clear();
}
+// This is where we cut the tree as described above. We also remember that
+// we might have to connect parent work with this work later.
+for (Operator? parent : new
ArrayListOperator?(root.getParentOperators())) {
+ if (LOG.isDebugEnabled()) {
+LOG.debug(Removing + parent + as parent from + root);
+ }
+ context.leafOperatorToFollowingWork.put(parent, work);
+ root.removeParent(parent);
+}
+
if (!context.currentUnionOperators.isEmpty()) {
// if there are union all operators we need to add the work to the set
// of union operators.
@@ -288,21 +298,6 @@ public class GenTezWork implements NodeP
work = unionWork;
}
-
-// This is where we cut the tree as described above. We also remember that
-// we might have to connect parent work with this work later.
-boolean removeParents = false;
-for (Operator? parent: new
ArrayListOperator?(root.getParentOperators())) {
- removeParents = true;
- context.leafOperatorToFollowingWork.put(parent, work);
- LOG.debug(Removing + parent + as parent from + root);
-}
-if (removeParents) {
- for (Operator? parent : new
ArrayListOperator?(root.getParentOperators())) {
-root.removeParent(parent);
- }
-}
-
// We're scanning a tree from roots to leaf (this is not technically
// correct, demux and mux operators might form a diamond shape, but
// we will only scan one path and ignore the others, because the
@@ -350,19 +345,14 @@ public class GenTezWork implements NodeP
// this can only be possible if there is merge work followed by the
union
UnionWork unionWork = (UnionWork) followingWork;
int index = getFollowingWorkIndex(tezWork, unionWork, rs);
- if (index != -1) {
-BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
-if (baseWork instanceof MergeJoinWork) {
- MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
- // disconnect the connection to union work and connect to merge
work
- followingWork = mergeJoinWork;
- rWork = (ReduceWork) mergeJoinWork.getMainWork();
-} else {
- rWork = (ReduceWork) baseWork;
-}
+ BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
+ if (baseWork instanceof MergeJoinWork) {
+MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
+// disconnect the connection to union work and connect to merge
work
+followingWork = mergeJoinWork;
+rWork = (ReduceWork) mergeJoinWork.getMainWork();
} else {
-throw new SemanticException(Following work not found for the
reduce sink:
-+ rs.getName());
+rWork = (ReduceWork) baseWork;
}
} else {
rWork = (ReduceWork) followingWork;
@@ -406,17 +396,17 @@ public
svn commit: r1652532 - in /hive/trunk: itests/src/test/resources/testconfiguration.properties ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Author: vikram
Date: Fri Jan 16 22:23:41 2015
New Revision: 1652532
URL: http://svn.apache.org/r1652532
Log:
HIVE-9038: Join tests fail on Tez (Vikram Dixit K, reviewed by Sergey Shelukhin)
Modified:
hive/trunk/itests/src/test/resources/testconfiguration.properties
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1652532r1=1652531r2=1652532view=diff
==
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Fri Jan
16 22:23:41 2015
@@ -271,7 +271,11 @@ minitez.query.files.shared=alter_merge_2
auto_sortmerge_join_5.q,\
auto_sortmerge_join_7.q,\
auto_sortmerge_join_8.q,\
- auto_sortmerge_join_9.q
+ auto_sortmerge_join_9.q,\
+ auto_join30.q,\
+ auto_join21.q,\
+ auto_join29.q,\
+ auto_join_filters.q
minitez.query.files=bucket_map_join_tez1.q,\
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1652532r1=1652531r2=1652532view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Fri Jan 16 22:23:41 2015
@@ -504,7 +504,38 @@ public class ConvertJoinMapJoin implemen
}
public int getMapJoinConversionPos(JoinOperator joinOp,
OptimizeTezProcContext context,
- int buckets) {
+ int buckets) throws SemanticException {
+/*
+ * HIVE-9038: Join tests fail in tez when we have more than 1 join on the
same key and there is
+ * an outer join down the join tree that requires filterTag. We disable
this conversion to map
+ * join here now. We need to emulate the behavior of HashTableSinkOperator
as in MR or create a
+ * new operation to be able to support this. This seems like a corner case
enough to special
+ * case this for now.
+ */
+if (joinOp.getConf().getConds().length 1) {
+ boolean hasOuter = false;
+ for (JoinCondDesc joinCondDesc : joinOp.getConf().getConds()) {
+switch (joinCondDesc.getType()) {
+case JoinDesc.INNER_JOIN:
+case JoinDesc.LEFT_SEMI_JOIN:
+case JoinDesc.UNIQUE_JOIN:
+ hasOuter = false;
+ break;
+
+case JoinDesc.FULL_OUTER_JOIN:
+case JoinDesc.LEFT_OUTER_JOIN:
+case JoinDesc.RIGHT_OUTER_JOIN:
+ hasOuter = true;
+ break;
+
+default:
+ throw new SemanticException(Unknown join type +
joinCondDesc.getType());
+}
+ }
+ if (hasOuter) {
+return -1;
+ }
+}
SetInteger bigTableCandidateSet =
MapJoinProcessor.getBigTableCandidates(joinOp.getConf().getConds());
svn commit: r1652535 [4/4] - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/results/clientpositive/tez/
Added: hive/trunk/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out?rev=1652535view=auto == --- hive/trunk/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out Fri Jan 16 22:28:00 2015 @@ -0,0 +1,540 @@ +PREHOOK: query: CREATE TABLE myinput1(key int, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@myinput1 +POSTHOOK: query: CREATE TABLE myinput1(key int, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@myinput1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1 +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: default@myinput1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1 +POSTHOOK: type: LOAD + A masked pattern was here +POSTHOOK: Output: default@myinput1 +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 + A masked pattern was here +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 + A masked pattern was here +3078400 +Warning: Map Join MAPJOIN[15][bigTable=a] in task 'Map 1' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 + A masked pattern was here +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 + A masked pattern was here +4937935 +Warning: Map Join MAPJOIN[15][bigTable=b] in task 'Map 2' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 + A masked pattern was here +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 + A masked pattern was here +3080335 +Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 + A masked pattern was here +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 + A masked pattern was here +4939870 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 + A masked pattern was here +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 + A masked pattern was here +3078400 +PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 + A masked pattern was here +POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON
svn commit: r1652535 [3/4] - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/results/clientpositive/tez/
Added: hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out?rev=1652535view=auto == --- hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out Fri Jan 16 22:28:00 2015 @@ -0,0 +1,1335 @@ +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Tez + Edges: +Reducer 2 - Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) +Reducer 3 - Reducer 2 (SIMPLE_EDGE) +Reducer 5 - Map 4 (SIMPLE_EDGE) + A masked pattern was here + Vertices: +Map 1 +Map Operator Tree: +TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col0 (type: string) +sort order: + +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Map 4 +Map Operator Tree: +TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col1 (type: string) +sort order: + +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: string) +Reducer 2 +Reduce Operator Tree: + Select Operator +expressions: KEY.reducesinkkey0 (type: string) +outputColumnNames: _col0 +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Map Join Operator + condition map: + Inner Join 0 to 1 + keys: +0 _col0 (type: string) +1 _col0 (type: string) + outputColumnNames: _col2, _col3 + input vertices: +1 Reducer 5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator +aggregations: sum(hash(_col2,_col3)) +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) +Reducer 3 +Reduce Operator Tree: + Group By Operator +aggregations: sum(VALUE._col0) +mode: mergepartial +outputColumnNames: _col0 +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator +
svn commit: r1652535 [2/4] - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/results/clientpositive/tez/
Added: hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out?rev=1652535view=auto
==
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out Fri Jan
16 22:28:00 2015
@@ -0,0 +1,3011 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key 10 AND src2.key 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key 10 AND src2.key 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+Tez
+ Edges:
+Reducer 2 - Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5
(SIMPLE_EDGE)
+Reducer 3 - Reducer 2 (SIMPLE_EDGE)
+ A masked pattern was here
+ Vertices:
+Map 1
+Map Operator Tree:
+TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+key expressions: key (type: string)
+sort order: +
+Map-reduce partition columns: key (type: string)
+Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+value expressions: value (type: string)
+Map 4
+Map Operator Tree:
+TableScan
+ alias: src2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+predicate: (key 10) (type: boolean)
+Statistics: Num rows: 166 Data size: 1763 Basic stats:
COMPLETE Column stats: NONE
+Reduce Output Operator
+ key expressions: key (type: string)
+ sort order: +
+ Map-reduce partition columns: key (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: value (type: string)
+Map 5
+Map Operator Tree:
+TableScan
+ alias: src3
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+key expressions: key (type: string)
+sort order: +
+Map-reduce partition columns: key (type: string)
+Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+value expressions: value (type: string)
+Reducer 2
+Reduce Operator Tree:
+ Merge Join Operator
+condition map:
+ Left Outer Join0 to 1
+ Right Outer Join1 to 2
+filter predicates:
+ 0 {(KEY.reducesinkkey0 10)}
+ 1
+ 2 {(KEY.reducesinkkey0 10)}
+keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ 2 key (type: string)
+outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+Statistics: Num rows: 1100 Data size: 11686 Basic stats:
COMPLETE Column stats: NONE
+Select Operator
+ expressions: _col0 (type: string), _col1 (type: string),
_col5 (type: string), _col6 (type: string), _col10 (type: string), _col11
(type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1100 Data size: 11686 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+key expressions: _col0 (type: string), _col1 (type:
string), _col2 (type: string), _col3 (type: string), _col4 (type: string),
_col5 (type: string)
+sort order: ++
+Statistics: Num rows: 1100 Data size: 11686 Basic stats:
COMPLETE Column stats: NONE
+Reducer 3
+Reduce Operator Tree:
+ Select Operator
+expressions: KEY.reducesinkkey0 (type: string),
svn commit: r1652535 [1/4] - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/results/clientpositive/tez/
Author: vikram
Date: Fri Jan 16 22:28:00 2015
New Revision: 1652535
URL: http://svn.apache.org/r1652535
Log:
HIVE-9038: Join tests fail on Tez (missed files in the previous commit) (Vikram
Dixit K, reviewed by Sergey Shelukhin)
Added:
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join21.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join29.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join30.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/auto_join_nulls.q.out
Modified:
hive/trunk/itests/src/test/resources/testconfiguration.properties
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1652535r1=1652534r2=1652535view=diff
==
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Fri Jan
16 22:28:00 2015
@@ -271,11 +271,7 @@ minitez.query.files.shared=alter_merge_2
auto_sortmerge_join_5.q,\
auto_sortmerge_join_7.q,\
auto_sortmerge_join_8.q,\
- auto_sortmerge_join_9.q,\
- auto_join30.q,\
- auto_join21.q,\
- auto_join29.q,\
- auto_join_filters.q
+ auto_sortmerge_join_9.q
minitez.query.files=bucket_map_join_tez1.q,\
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1652535r1=1652534r2=1652535view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Fri Jan 16 22:28:00 2015
@@ -504,38 +504,7 @@ public class ConvertJoinMapJoin implemen
}
public int getMapJoinConversionPos(JoinOperator joinOp,
OptimizeTezProcContext context,
- int buckets) throws SemanticException {
-/*
- * HIVE-9038: Join tests fail in tez when we have more than 1 join on the
same key and there is
- * an outer join down the join tree that requires filterTag. We disable
this conversion to map
- * join here now. We need to emulate the behavior of HashTableSinkOperator
as in MR or create a
- * new operation to be able to support this. This seems like a corner case
enough to special
- * case this for now.
- */
-if (joinOp.getConf().getConds().length 1) {
- boolean hasOuter = false;
- for (JoinCondDesc joinCondDesc : joinOp.getConf().getConds()) {
-switch (joinCondDesc.getType()) {
-case JoinDesc.INNER_JOIN:
-case JoinDesc.LEFT_SEMI_JOIN:
-case JoinDesc.UNIQUE_JOIN:
- hasOuter = false;
- break;
-
-case JoinDesc.FULL_OUTER_JOIN:
-case JoinDesc.LEFT_OUTER_JOIN:
-case JoinDesc.RIGHT_OUTER_JOIN:
- hasOuter = true;
- break;
-
-default:
- throw new SemanticException(Unknown join type +
joinCondDesc.getType());
-}
- }
- if (hasOuter) {
-return -1;
- }
-}
+ int buckets) {
SetInteger bigTableCandidateSet =
MapJoinProcessor.getBigTableCandidates(joinOp.getConf().getConds());
Added: hive/trunk/ql/src/test/results/clientpositive/tez/auto_join21.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/auto_join21.q.out?rev=1652535view=auto
==
--- hive/trunk/ql/src/test/results/clientpositive/tez/auto_join21.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/auto_join21.q.out Fri Jan
16 22:28:00 2015
@@ -0,0 +1,606 @@
+PREHOOK: query: explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key 10 AND src2.key 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key 10 AND src2.key 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+Tez
+ Edges:
+Reducer 2 - Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5
(SIMPLE_EDGE)
+Reducer 3 - Reducer 2 (SIMPLE_EDGE
svn commit: r1652536 [3/4] - in /hive/branches/branch-0.14: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/results/clientpositive/tez/
Added: hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_join30.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_join30.q.out?rev=1652536view=auto == --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_join30.q.out (added) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_join30.q.out Fri Jan 16 22:30:17 2015 @@ -0,0 +1,1335 @@ +PREHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM +(SELECT src.* FROM src sort by key) x +JOIN +(SELECT src.* FROM src sort by value) Y +ON (x.key = Y.key) +select sum(hash(Y.key,Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Tez + Edges: +Reducer 2 - Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) +Reducer 3 - Reducer 2 (SIMPLE_EDGE) +Reducer 5 - Map 4 (SIMPLE_EDGE) + A masked pattern was here + Vertices: +Map 1 +Map Operator Tree: +TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col0 (type: string) +sort order: + +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Map 4 +Map Operator Tree: +TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col1 (type: string) +sort order: + +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: string) +Reducer 2 +Reduce Operator Tree: + Select Operator +expressions: KEY.reducesinkkey0 (type: string) +outputColumnNames: _col0 +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Map Join Operator + condition map: + Inner Join 0 to 1 + keys: +0 _col0 (type: string) +1 _col0 (type: string) + outputColumnNames: _col2, _col3 + input vertices: +1 Reducer 5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator +aggregations: sum(hash(_col2,_col3)) +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) +Reducer 3 +Reduce Operator Tree: + Group By Operator +aggregations: sum(VALUE._col0) +mode: mergepartial +outputColumnNames: _col0 +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
svn commit: r1652536 [2/4] - in /hive/branches/branch-0.14: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/results/clientpositive/tez/
Added:
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_join29.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_join29.q.out?rev=1652536view=auto
==
---
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_join29.q.out
(added)
+++
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_join29.q.out
Fri Jan 16 22:30:17 2015
@@ -0,0 +1,3011 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key 10 AND src2.key 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key 10 AND src2.key 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+Tez
+ Edges:
+Reducer 2 - Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5
(SIMPLE_EDGE)
+Reducer 3 - Reducer 2 (SIMPLE_EDGE)
+ A masked pattern was here
+ Vertices:
+Map 1
+Map Operator Tree:
+TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+key expressions: key (type: string)
+sort order: +
+Map-reduce partition columns: key (type: string)
+Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+value expressions: value (type: string)
+Map 4
+Map Operator Tree:
+TableScan
+ alias: src2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+predicate: (key 10) (type: boolean)
+Statistics: Num rows: 166 Data size: 1763 Basic stats:
COMPLETE Column stats: NONE
+Reduce Output Operator
+ key expressions: key (type: string)
+ sort order: +
+ Map-reduce partition columns: key (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: value (type: string)
+Map 5
+Map Operator Tree:
+TableScan
+ alias: src3
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+key expressions: key (type: string)
+sort order: +
+Map-reduce partition columns: key (type: string)
+Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+value expressions: value (type: string)
+Reducer 2
+Reduce Operator Tree:
+ Merge Join Operator
+condition map:
+ Left Outer Join0 to 1
+ Right Outer Join1 to 2
+filter predicates:
+ 0 {(KEY.reducesinkkey0 10)}
+ 1
+ 2 {(KEY.reducesinkkey0 10)}
+keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ 2 key (type: string)
+outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+Statistics: Num rows: 1100 Data size: 11686 Basic stats:
COMPLETE Column stats: NONE
+Select Operator
+ expressions: _col0 (type: string), _col1 (type: string),
_col5 (type: string), _col6 (type: string), _col10 (type: string), _col11
(type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1100 Data size: 11686 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+key expressions: _col0 (type: string), _col1 (type:
string), _col2 (type: string), _col3 (type: string), _col4 (type: string),
_col5 (type: string)
+sort order: ++
+Statistics: Num rows: 1100 Data size: 11686 Basic stats:
COMPLETE Column stats: NONE
+Reducer 3
+Reduce Operator Tree:
+ Select Operator
+
svn commit: r1653426 - /hive/branches/branch-1.0/
Author: vikram Date: Wed Jan 21 03:01:10 2015 New Revision: 1653426 URL: http://svn.apache.org/r1653426 Log: Creating the 1.0 branch from branch 0.14 Added: hive/branches/branch-1.0/ (props changed) - copied from r1653425, hive/branches/branch-0.14/ Propchange: hive/branches/branch-1.0/ -- reviewboard:url = https://reviews.apache.org Propchange: hive/branches/branch-1.0/ -- --- svn:ignore (added) +++ svn:ignore Wed Jan 21 03:01:10 2015 @@ -0,0 +1,11 @@ +datanucleus.log +eclipse-build +target +.arc_jira_lib +.classpath* +.externalToolBuilders +.project +.settings +*.launch +*~ +metastore_db Propchange: hive/branches/branch-1.0/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Wed Jan 21 03:01:10 2015 @@ -0,0 +1,5 @@ +/hive/branches/branch-0.11:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184 +/hive/branches/cbo:1605012-1627125 +/hive/branches/tez:1494760-1622766 +/hive/branches/vectorization:1466908-1527856 +/hive/trunk:1627830,1629752,1630512,1635373,1637170,1639249
svn commit: r1654968 [2/2] - in /hive/branches/branch-1.0/ql/src/test/results/clientpositive: ./ tez/
Modified: hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out?rev=1654968r1=1654967r2=1654968view=diff == --- hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out (original) +++ hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out Tue Jan 27 03:13:58 2015 @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH POSTHOOK: type: LOAD A masked pattern was here POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 A masked pattern was here 3078400 -Warning: Map Join MAPJOIN[15][bigTable=a] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[15][bigTable=a] in task 'Map 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -34,7 +34,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 A masked pattern was here 4937935 -Warning: Map Join MAPJOIN[15][bigTable=b] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[15][bigTable=b] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -44,7 +44,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 A masked pattern was here 3080335 -Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [a, b]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -300,7 +300,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH POSTHOOK: type: LOAD A masked pattern was here POSTHOOK: Output: default@smb_input2 -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -310,7 +310,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 A masked pattern was here 3078400 -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -320,7 +320,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 A masked pattern was here 3078400 -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -330,7 +330,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 A masked pattern was here 3078400 -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [a, b]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key 40 AND a.value 50 AND a.key = a.value AND b.key 40 AND b.value 50 AND b.key = b.value PREHOOK: type:
svn commit: r1654968 [1/2] - in /hive/branches/branch-1.0/ql/src/test/results/clientpositive: ./ tez/
Author: vikram Date: Tue Jan 27 03:13:58 2015 New Revision: 1654968 URL: http://svn.apache.org/r1654968 Log: Branch 1.0: Missing golden file updates from trunk Modified: hive/branches/branch-1.0/ql/src/test/results/clientpositive/constprog2.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/quotedid_basic.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/regex_col.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/auto_join21.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/auto_join29.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/auto_join30.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/auto_join_filters.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/tez_union2.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vector_coalesce_2.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/vector_coalesce_2.q.out hive/branches/branch-1.0/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out Modified: hive/branches/branch-1.0/ql/src/test/results/clientpositive/constprog2.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/test/results/clientpositive/constprog2.q.out?rev=1654968r1=1654967r2=1654968view=diff == --- hive/branches/branch-1.0/ql/src/test/results/clientpositive/constprog2.q.out (original) +++ hive/branches/branch-1.0/ql/src/test/results/clientpositive/constprog2.q.out Tue Jan 27 03:13:58 2015 @@ -23,7 +23,7 @@ STAGE PLANS: Reduce Output Operator key expressions: '86' (type: string) sort order: + -Map-reduce partition columns: '' (type: string) +Map-reduce partition columns: '86' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -35,7 +35,7 @@ STAGE PLANS: Reduce Output Operator key expressions: '86' (type: string) sort order: + -Map-reduce partition columns: '' (type: string) +Map-reduce partition columns: '86' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator @@ -100,7 +100,7 @@ STAGE PLANS: Reduce Output Operator key expressions: '86' (type: string) sort order: + -Map-reduce partition columns: '' (type: string) +Map-reduce partition columns: '86' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan @@ -112,7 +112,7 @@ STAGE PLANS: Reduce Output Operator key expressions: '86' (type: string) sort order: + -Map-reduce partition columns: '' (type: string) +Map-reduce partition columns: '86' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator Modified: hive/branches/branch-1.0/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out?rev=1654968r1=1654967r2=1654968view=diff == --- hive/branches/branch-1.0/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out (original) +++ hive/branches/branch-1.0/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out Tue Jan 27 03:13:58 2015 @@ -191,7 +191,7 @@ analyze table orc_merge5b compute statis POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_merge5b POSTHOOK: Output: default@orc_merge5b -Found 3 items +Found 4 items A masked pattern was here PREHOOK: query: select * from orc_merge5b PREHOOK: type: QUERY Modified: hive/branches/branch-1.0/ql/src/test/results/clientpositive/quotedid_basic.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/test/results/clientpositive
svn commit: r1654918 - in /hive/branches/branch-1.0: ./ accumulo-handler/ ant/ beeline/ cli/ common/ contrib/ hbase-handler/ hcatalog/ hcatalog/core/ hcatalog/hcatalog-pig-adapter/ hcatalog/server-ext
Author: vikram Date: Tue Jan 27 00:00:45 2015 New Revision: 1654918 URL: http://svn.apache.org/r1654918 Log: Branch 1.0: Renaming the version in all pom.xml files Modified: hive/branches/branch-1.0/accumulo-handler/pom.xml hive/branches/branch-1.0/ant/pom.xml hive/branches/branch-1.0/beeline/pom.xml hive/branches/branch-1.0/cli/pom.xml hive/branches/branch-1.0/common/pom.xml hive/branches/branch-1.0/contrib/pom.xml hive/branches/branch-1.0/hbase-handler/pom.xml hive/branches/branch-1.0/hcatalog/core/pom.xml hive/branches/branch-1.0/hcatalog/hcatalog-pig-adapter/pom.xml hive/branches/branch-1.0/hcatalog/pom.xml hive/branches/branch-1.0/hcatalog/server-extensions/pom.xml hive/branches/branch-1.0/hcatalog/streaming/pom.xml hive/branches/branch-1.0/hcatalog/webhcat/java-client/pom.xml hive/branches/branch-1.0/hcatalog/webhcat/svr/pom.xml hive/branches/branch-1.0/hwi/pom.xml hive/branches/branch-1.0/itests/custom-serde/pom.xml hive/branches/branch-1.0/itests/hcatalog-unit/pom.xml hive/branches/branch-1.0/itests/hive-minikdc/pom.xml hive/branches/branch-1.0/itests/hive-unit-hadoop2/pom.xml hive/branches/branch-1.0/itests/hive-unit/pom.xml hive/branches/branch-1.0/itests/pom.xml hive/branches/branch-1.0/itests/qtest/pom.xml hive/branches/branch-1.0/itests/test-serde/pom.xml hive/branches/branch-1.0/itests/util/pom.xml hive/branches/branch-1.0/jdbc/pom.xml hive/branches/branch-1.0/metastore/pom.xml hive/branches/branch-1.0/odbc/pom.xml hive/branches/branch-1.0/packaging/pom.xml hive/branches/branch-1.0/pom.xml hive/branches/branch-1.0/ql/pom.xml hive/branches/branch-1.0/serde/pom.xml hive/branches/branch-1.0/service/pom.xml hive/branches/branch-1.0/shims/0.20/pom.xml hive/branches/branch-1.0/shims/0.20S/pom.xml hive/branches/branch-1.0/shims/0.23/pom.xml hive/branches/branch-1.0/shims/aggregator/pom.xml hive/branches/branch-1.0/shims/common-secure/pom.xml hive/branches/branch-1.0/shims/common/pom.xml hive/branches/branch-1.0/shims/pom.xml hive/branches/branch-1.0/testutils/pom.xml Modified: hive/branches/branch-1.0/accumulo-handler/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/accumulo-handler/pom.xml?rev=1654918r1=1654917r2=1654918view=diff == --- hive/branches/branch-1.0/accumulo-handler/pom.xml (original) +++ hive/branches/branch-1.0/accumulo-handler/pom.xml Tue Jan 27 00:00:45 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/ant/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ant/pom.xml?rev=1654918r1=1654917r2=1654918view=diff == --- hive/branches/branch-1.0/ant/pom.xml (original) +++ hive/branches/branch-1.0/ant/pom.xml Tue Jan 27 00:00:45 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/beeline/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/beeline/pom.xml?rev=1654918r1=1654917r2=1654918view=diff == --- hive/branches/branch-1.0/beeline/pom.xml (original) +++ hive/branches/branch-1.0/beeline/pom.xml Tue Jan 27 00:00:45 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/cli/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/cli/pom.xml?rev=1654918r1=1654917r2=1654918view=diff == --- hive/branches/branch-1.0/cli/pom.xml (original) +++ hive/branches/branch-1.0/cli/pom.xml Tue Jan 27 00:00:45 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/common/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/common/pom.xml?rev=1654918r1=1654917r2=1654918view=diff == --- hive/branches/branch-1.0/common/pom.xml (original) +++ hive/branches/branch-1.0/common/pom.xml Tue Jan 27 00:00:45 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version
svn commit: r1654920 - in /hive/branches/branch-1.0: ./ accumulo-handler/ ant/ beeline/ cli/ common/ contrib/ hbase-handler/ hcatalog/ hcatalog/core/ hcatalog/hcatalog-pig-adapter/ hcatalog/server-ext
Author: vikram Date: Tue Jan 27 00:11:29 2015 New Revision: 1654920 URL: http://svn.apache.org/r1654920 Log: Branch 1.0: Renaming the version in all pom.xml files Modified: hive/branches/branch-1.0/accumulo-handler/pom.xml hive/branches/branch-1.0/ant/pom.xml hive/branches/branch-1.0/beeline/pom.xml hive/branches/branch-1.0/cli/pom.xml hive/branches/branch-1.0/common/pom.xml hive/branches/branch-1.0/contrib/pom.xml hive/branches/branch-1.0/hbase-handler/pom.xml hive/branches/branch-1.0/hcatalog/core/pom.xml hive/branches/branch-1.0/hcatalog/hcatalog-pig-adapter/pom.xml hive/branches/branch-1.0/hcatalog/pom.xml hive/branches/branch-1.0/hcatalog/server-extensions/pom.xml hive/branches/branch-1.0/hcatalog/streaming/pom.xml hive/branches/branch-1.0/hcatalog/webhcat/java-client/pom.xml hive/branches/branch-1.0/hcatalog/webhcat/svr/pom.xml hive/branches/branch-1.0/hwi/pom.xml hive/branches/branch-1.0/itests/custom-serde/pom.xml hive/branches/branch-1.0/itests/hcatalog-unit/pom.xml hive/branches/branch-1.0/itests/hive-minikdc/pom.xml hive/branches/branch-1.0/itests/hive-unit-hadoop2/pom.xml hive/branches/branch-1.0/itests/hive-unit/pom.xml hive/branches/branch-1.0/itests/pom.xml hive/branches/branch-1.0/itests/qtest/pom.xml hive/branches/branch-1.0/itests/test-serde/pom.xml hive/branches/branch-1.0/itests/util/pom.xml hive/branches/branch-1.0/jdbc/pom.xml hive/branches/branch-1.0/metastore/pom.xml hive/branches/branch-1.0/odbc/pom.xml hive/branches/branch-1.0/packaging/pom.xml hive/branches/branch-1.0/pom.xml hive/branches/branch-1.0/ql/pom.xml hive/branches/branch-1.0/serde/pom.xml hive/branches/branch-1.0/service/pom.xml hive/branches/branch-1.0/shims/0.20/pom.xml hive/branches/branch-1.0/shims/0.20S/pom.xml hive/branches/branch-1.0/shims/0.23/pom.xml hive/branches/branch-1.0/shims/aggregator/pom.xml hive/branches/branch-1.0/shims/common-secure/pom.xml hive/branches/branch-1.0/shims/common/pom.xml hive/branches/branch-1.0/shims/pom.xml hive/branches/branch-1.0/testutils/pom.xml Modified: hive/branches/branch-1.0/accumulo-handler/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/accumulo-handler/pom.xml?rev=1654920r1=1654919r2=1654920view=diff == --- hive/branches/branch-1.0/accumulo-handler/pom.xml (original) +++ hive/branches/branch-1.0/accumulo-handler/pom.xml Tue Jan 27 00:11:29 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/ant/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ant/pom.xml?rev=1654920r1=1654919r2=1654920view=diff == --- hive/branches/branch-1.0/ant/pom.xml (original) +++ hive/branches/branch-1.0/ant/pom.xml Tue Jan 27 00:11:29 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/beeline/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/beeline/pom.xml?rev=1654920r1=1654919r2=1654920view=diff == --- hive/branches/branch-1.0/beeline/pom.xml (original) +++ hive/branches/branch-1.0/beeline/pom.xml Tue Jan 27 00:11:29 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/cli/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/cli/pom.xml?rev=1654920r1=1654919r2=1654920view=diff == --- hive/branches/branch-1.0/cli/pom.xml (original) +++ hive/branches/branch-1.0/cli/pom.xml Tue Jan 27 00:11:29 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/common/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/common/pom.xml?rev=1654920r1=1654919r2=1654920view=diff == --- hive/branches/branch-1.0/common/pom.xml (original) +++ hive/branches/branch-1.0/common/pom.xml Tue Jan 27 00:11:29 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version0.14.0/version +version1.0/version
svn commit: r1655172 - in /hive/branches/branch-1.0/ql/src: java/org/apache/hadoop/hive/ql/optimizer/physical/ test/results/clientpositive/ test/results/clientpositive/tez/
Author: vikram
Date: Tue Jan 27 22:09:59 2015
New Revision: 1655172
URL: http://svn.apache.org/r1655172
Log:
HIVE-9053 : select constant in union all followed by group by gives wrong
result (Pengcheng Xiong via Vikram Dixit)
Modified:
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
hive/branches/branch-1.0/ql/src/test/results/clientpositive/metadataonly1.q.out
hive/branches/branch-1.0/ql/src/test/results/clientpositive/optimize_nullscan.q.out
hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out
Modified:
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java?rev=1655172r1=1655171r2=1655172view=diff
==
---
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
(original)
+++
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
Tue Jan 27 22:09:59 2015
@@ -133,8 +133,7 @@ public class MetadataOnlyOptimizer imple
WalkerCtx walkerCtx = (WalkerCtx) procCtx;
// There can be atmost one element eligible to be converted to
// metadata only
- if ((walkerCtx.getMayBeMetadataOnlyTableScans().isEmpty())
- || (walkerCtx.getMayBeMetadataOnlyTableScans().size() 1)) {
+ if (walkerCtx.getMayBeMetadataOnlyTableScans().isEmpty()) {
return nd;
}
Modified:
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java?rev=1655172r1=1655171r2=1655172view=diff
==
---
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
(original)
+++
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
Tue Jan 27 22:09:59 2015
@@ -21,9 +21,11 @@ package org.apache.hadoop.hive.ql.optimi
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Iterator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Stack;
import org.apache.commons.logging.Log;
@@ -100,31 +102,51 @@ public class NullScanTaskDispatcher impl
return paths;
}
-
- private void processAlias(MapWork work, String alias) {
-ListString paths = getPathsForAlias(work, alias);
-if (paths.isEmpty()) {
- // partitioned table which don't select any partitions
- // there are no paths to replace with fakePath
- return;
-}
-work.setUseOneNullRowInputFormat(true);
-
-// Change the alias partition desc
-PartitionDesc aliasPartn = work.getAliasToPartnInfo().get(alias);
-changePartitionToMetadataOnly(aliasPartn);
-
-
-for (String path : paths) {
- PartitionDesc partDesc = work.getPathToPartitionInfo().get(path);
+
+ private void processAlias(MapWork work, String path, ArrayListString
aliasesAffected,
+ ArrayListString aliases) {
+// the aliases that are allowed to map to a null scan.
+ArrayListString allowed = new ArrayListString();
+for (String alias : aliasesAffected) {
+ if (aliases.contains(alias)) {
+allowed.add(alias);
+ }
+}
+if (allowed.size() 0) {
+ work.setUseOneNullRowInputFormat(true);
+ PartitionDesc partDesc = work.getPathToPartitionInfo().get(path).clone();
PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc);
Path fakePath = new Path(physicalContext.getContext().getMRTmpPath()
- + newPartition.getTableName()
- + encode(newPartition.getPartSpec()));
- work.getPathToPartitionInfo().remove(path);
+ + newPartition.getTableName() + encode(newPartition.getPartSpec()));
work.getPathToPartitionInfo().put(fakePath.getName(), newPartition);
- ArrayListString aliases = work.getPathToAliases().remove(path);
- work.getPathToAliases().put(fakePath.getName(), aliases);
+ work.getPathToAliases().put(fakePath.getName(), new
ArrayListString(allowed));
+ aliasesAffected.removeAll(allowed);
+ if (aliasesAffected.isEmpty()) {
+work.getPathToAliases().remove(path
svn commit: r1655176 - /hive/branches/branch-1.0/RELEASE_NOTES.txt
Author: vikram Date: Tue Jan 27 22:13:40 2015 New Revision: 1655176 URL: http://svn.apache.org/r1655176 Log: Branch 1.0: Update release notes Modified: hive/branches/branch-1.0/RELEASE_NOTES.txt Modified: hive/branches/branch-1.0/RELEASE_NOTES.txt URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/RELEASE_NOTES.txt?rev=1655176r1=1655175r2=1655176view=diff == --- hive/branches/branch-1.0/RELEASE_NOTES.txt (original) +++ hive/branches/branch-1.0/RELEASE_NOTES.txt Tue Jan 27 22:13:40 2015 @@ -1,3 +1,72 @@ +Release Notes - Hive - Version 1.0.0 + +** Bug +* [HIVE-5631] - Index creation on a skew table fails +* [HIVE-5664] - Drop cascade database fails when the db has any tables with indexes +* [HIVE-6198] - ORC file and struct column names are case sensitive +* [HIVE-6468] - HS2 Metastore using SASL out of memory error when curl sends a get request +* [HIVE-7270] - SerDe Properties are not considered by show create table Command +* [HIVE-8099] - IN operator for partition column fails when the partition column type is DATE +* [HIVE-8295] - Add batch retrieve partition objects for metastore direct sql +* [HIVE-8374] - schematool fails on Postgres versions 9.2 +* [HIVE-8485] - HMS on Oracle incompatibility +* [HIVE-8706] - Table statistic collection on counter failed due to table name character case. +* [HIVE-8715] - Hive 14 upgrade scripts can fail for statistics if database was created using auto-create +* [HIVE-8739] - handle Derby and Oracle errors with joins and filters in Direct SQL in a invalid-DB-specific path +* [HIVE-8784] - Querying partition does not work with JDO enabled against PostgreSQL +* [HIVE-8805] - CBO skipped due to SemanticException: Line 0:-1 Both left and right aliases encountered in JOIN 'avg_cs_ext_discount_amt' +* [HIVE-8811] - Dynamic partition pruning can result in NPE during query compilation +* [HIVE-8827] - Remove SSLv2Hello from list of disabled protocols +* [HIVE-8830] - hcatalog process don't exit because of non daemon thread +* [HIVE-8845] - Switch to Tez 0.5.2 +* [HIVE-8866] - Vectorization on partitioned table throws ArrayIndexOutOfBoundsException when partitions are not of same #of columns +* [HIVE-8870] - errors when selecting a struct field within an array from ORC based tables +* [HIVE-8873] - Switch to calcite 0.9.2 +* [HIVE-8876] - incorrect upgrade script for Oracle (13-14) +* [HIVE-8880] - non-synchronized access to split list in OrcInputFormat +* [HIVE-8886] - Some Vectorized String CONCAT expressions result in runtime error Vectorization: Unsuported vector output type: StringGroup +* [HIVE-] - Mapjoin with LateralViewJoin generates wrong plan in Tez +* [HIVE-8947] - HIVE-8876 also affects Postgres 9.2 +* [HIVE-8966] - Delta files created by hive hcatalog streaming cannot be compacted +* [HIVE-9003] - Vectorized IF expr broken for the scalar and scalar case +* [HIVE-9025] - join38.q (without map join) produces incorrect result when testing with multiple reducers +* [HIVE-9038] - Join tests fail on Tez +* [HIVE-9051] - TezJobMonitor in-place updates logs too often to logfile +* [HIVE-9053] - select constant in union all followed by group by gives wrong result +* [HIVE-9067] - OrcFileMergeOperator may create merge file that does not match properties of input files +* [HIVE-9090] - Rename Tez File Merge Work to smaller name +* [HIVE-9108] - Fix for HIVE-8735 is incorrect (stats with long paths) +* [HIVE-9111] - Potential NPE in OrcStruct for list and map types +* [HIVE-9112] - Query may generate different results depending on the number of reducers +* [HIVE-9114] - union all query in cbo test has undefined ordering +* [HIVE-9126] - Backport HIVE-8827 (Remove SSLv2Hello from list of disabled protocols) to 0.14 branch +* [HIVE-9141] - HiveOnTez: mix of union all, distinct, group by generates error +* [HIVE-9155] - HIVE_LOCKS uses int instead of bigint hive-txn-schema-0.14.0.mssql.sql +* [HIVE-9162] - stats19 test is environment-dependant +* [HIVE-9166] - Place an upper bound for SARG CNF conversion +* [HIVE-9168] - Vectorized Coalesce for strings is broken +* [HIVE-9205] - Change default tez install directory to use /tmp instead of /user and create the directory if it does not exist +* [HIVE-9234] - HiveServer2 leaks FileSystem objects in FileSystem.CACHE +* [HIVE-9249] - java.lang.ClassCastException: org.apache.hadoop.hive.serde2.io.HiveVarcharWritable cannot be cast to org.apache.hadoop.hive.common.type.HiveVarchar when joining tables +* [HIVE-9278] - Cached expression feature broken in one case +* [HIVE-9359] - Export of a large table causes OOM in Metastore and Client +* [HIVE-9361] - Intermittent NPE in SessionHiveMetaStoreClient.alterTempTable
svn commit: r1655177 - in /hive/branches/branch-1.0/ql/src/test: queries/clientpositive/metadataOnlyOptimizer.q results/clientpositive/metadataOnlyOptimizer.q.out
Author: vikram Date: Tue Jan 27 22:14:27 2015 New Revision: 1655177 URL: http://svn.apache.org/r1655177 Log: Missing files from previous commit HIVE-9053: select constant in union all followed by group by gives wrong result (Pengcheng Xiong via Vikram Dixit) Added: hive/branches/branch-1.0/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q hive/branches/branch-1.0/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out Added: hive/branches/branch-1.0/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q?rev=1655177view=auto == --- hive/branches/branch-1.0/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q (added) +++ hive/branches/branch-1.0/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q Tue Jan 27 22:14:27 2015 @@ -0,0 +1,44 @@ +select key from( +select '1' as key from srcpart where ds=2008-04-09 +UNION all +SELECT key from srcpart where ds=2008-04-09 and hr=11 +) tab group by key; + +select key from( +SELECT '1' as key from src +UNION all +SELECT key as key from src +) tab group by key; + +select max(key) from( +SELECT '1' as key from src +UNION all +SELECT key as key from src +) tab group by key; + +select key from( +SELECT '1' as key from src +UNION all +SELECT '2' as key from src +) tab group by key; + + +select key from( +SELECT '1' as key from src +UNION all +SELECT key as key from src +UNION all +SELECT '2' as key from src +UNION all +SELECT key as key from src +) tab group by key; + +select k from (SELECT '1' as k from src limit 0 union all select key as k from src limit 1)tab; + +select k from (SELECT '1' as k from src limit 1 union all select key as k from src limit 0)tab; + +select max(ds) from srcpart; + +select count(ds) from srcpart; + + Added: hive/branches/branch-1.0/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out?rev=1655177view=auto == --- hive/branches/branch-1.0/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out (added) +++ hive/branches/branch-1.0/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out Tue Jan 27 22:14:27 2015 @@ -0,0 +1,1386 @@ +PREHOOK: query: select key from( +select '1' as key from srcpart where ds=2008-04-09 +UNION all +SELECT key from srcpart where ds=2008-04-09 and hr=11 +) tab group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 + A masked pattern was here +POSTHOOK: query: select key from( +select '1' as key from srcpart where ds=2008-04-09 +UNION all +SELECT key from srcpart where ds=2008-04-09 and hr=11 +) tab group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 + A masked pattern was here +0 +1 +10 +100 +103 +104 +105 +11 +111 +113 +114 +116 +118 +119 +12 +120 +125 +126 +128 +129 +131 +133 +134 +136 +137 +138 +143 +145 +146 +149 +15 +150 +152 +153 +155 +156 +157 +158 +160 +162 +163 +164 +165 +166 +167 +168 +169 +17 +170 +172 +174 +175 +176 +177 +178 +179 +18 +180 +181 +183 +186 +187 +189 +19 +190 +191 +192 +193 +194 +195 +196 +197 +199 +2 +20 +200 +201 +202 +203 +205 +207 +208 +209 +213 +214 +216 +217 +218 +219 +221 +222 +223 +224 +226 +228 +229 +230 +233 +235 +237 +238 +239 +24 +241 +242 +244 +247 +248 +249 +252 +255 +256 +257 +258 +26 +260 +262 +263 +265 +266 +27 +272 +273 +274 +275 +277 +278 +28 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +291 +292 +296 +298 +30 +302 +305 +306 +307 +308 +309 +310 +311 +315 +316 +317 +318 +321 +322 +323 +325 +327 +33 +331 +332 +333 +335 +336 +338 +339 +34 +341 +342 +344 +345 +348 +35 +351 +353 +356 +360 +362 +364 +365 +366 +367 +368 +369 +37 +373 +374 +375 +377 +378 +379 +382 +384 +386 +389 +392 +393 +394 +395 +396 +397 +399 +4 +400 +401 +402 +403 +404 +406 +407 +409 +41 +411 +413 +414 +417 +418 +419 +42 +421 +424 +427 +429 +43 +430 +431 +432 +435 +436 +437 +438 +439 +44 +443 +444 +446 +448 +449 +452 +453 +454 +455 +457 +458 +459 +460 +462 +463 +466 +467 +468 +469 +47 +470 +472 +475 +477 +478 +479 +480 +481 +482 +483 +484 +485 +487 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +5 +51 +53 +54 +57 +58 +64 +65 +66 +67 +69 +70 +72 +74 +76 +77 +78 +8 +80 +82 +83 +84 +85 +86 +87 +9 +90 +92 +95 +96 +97 +98 +PREHOOK: query: select key from( +SELECT '1' as key from src +UNION all +SELECT key as key from src +) tab group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src + A masked pattern was here +POSTHOOK: query: select key from( +SELECT '1' as key from src
svn commit: r1654919 - in /hive/branches/branch-1.0: ./ accumulo-handler/ ant/ beeline/ cli/ common/ contrib/ hbase-handler/ hcatalog/ hcatalog/core/ hcatalog/hcatalog-pig-adapter/ hcatalog/server-ext
Author: vikram Date: Tue Jan 27 00:05:10 2015 New Revision: 1654919 URL: http://svn.apache.org/r1654919 Log: Revert version update because of unnecessary change commit Modified: hive/branches/branch-1.0/accumulo-handler/pom.xml hive/branches/branch-1.0/ant/pom.xml hive/branches/branch-1.0/beeline/pom.xml hive/branches/branch-1.0/cli/pom.xml hive/branches/branch-1.0/common/pom.xml hive/branches/branch-1.0/contrib/pom.xml hive/branches/branch-1.0/hbase-handler/pom.xml hive/branches/branch-1.0/hcatalog/core/pom.xml hive/branches/branch-1.0/hcatalog/hcatalog-pig-adapter/pom.xml hive/branches/branch-1.0/hcatalog/pom.xml hive/branches/branch-1.0/hcatalog/server-extensions/pom.xml hive/branches/branch-1.0/hcatalog/streaming/pom.xml hive/branches/branch-1.0/hcatalog/webhcat/java-client/pom.xml hive/branches/branch-1.0/hcatalog/webhcat/svr/pom.xml hive/branches/branch-1.0/hwi/pom.xml hive/branches/branch-1.0/itests/custom-serde/pom.xml hive/branches/branch-1.0/itests/hcatalog-unit/pom.xml hive/branches/branch-1.0/itests/hive-minikdc/pom.xml hive/branches/branch-1.0/itests/hive-unit-hadoop2/pom.xml hive/branches/branch-1.0/itests/hive-unit/pom.xml hive/branches/branch-1.0/itests/pom.xml hive/branches/branch-1.0/itests/qtest/pom.xml hive/branches/branch-1.0/itests/test-serde/pom.xml hive/branches/branch-1.0/itests/util/pom.xml hive/branches/branch-1.0/jdbc/pom.xml hive/branches/branch-1.0/metastore/pom.xml hive/branches/branch-1.0/odbc/pom.xml hive/branches/branch-1.0/packaging/pom.xml hive/branches/branch-1.0/pom.xml hive/branches/branch-1.0/ql/pom.xml hive/branches/branch-1.0/serde/pom.xml hive/branches/branch-1.0/service/pom.xml hive/branches/branch-1.0/shims/0.20/pom.xml hive/branches/branch-1.0/shims/0.20S/pom.xml hive/branches/branch-1.0/shims/0.23/pom.xml hive/branches/branch-1.0/shims/aggregator/pom.xml hive/branches/branch-1.0/shims/common-secure/pom.xml hive/branches/branch-1.0/shims/common/pom.xml hive/branches/branch-1.0/shims/pom.xml hive/branches/branch-1.0/testutils/pom.xml Modified: hive/branches/branch-1.0/accumulo-handler/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/accumulo-handler/pom.xml?rev=1654919r1=1654918r2=1654919view=diff == --- hive/branches/branch-1.0/accumulo-handler/pom.xml (original) +++ hive/branches/branch-1.0/accumulo-handler/pom.xml Tue Jan 27 00:05:10 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version1.0/version +version0.14.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/ant/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/ant/pom.xml?rev=1654919r1=1654918r2=1654919view=diff == --- hive/branches/branch-1.0/ant/pom.xml (original) +++ hive/branches/branch-1.0/ant/pom.xml Tue Jan 27 00:05:10 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version1.0/version +version0.14.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/beeline/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/beeline/pom.xml?rev=1654919r1=1654918r2=1654919view=diff == --- hive/branches/branch-1.0/beeline/pom.xml (original) +++ hive/branches/branch-1.0/beeline/pom.xml Tue Jan 27 00:05:10 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version1.0/version +version0.14.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/cli/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/cli/pom.xml?rev=1654919r1=1654918r2=1654919view=diff == --- hive/branches/branch-1.0/cli/pom.xml (original) +++ hive/branches/branch-1.0/cli/pom.xml Tue Jan 27 00:05:10 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version1.0/version +version0.14.0/version relativePath../pom.xml/relativePath /parent Modified: hive/branches/branch-1.0/common/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/common/pom.xml?rev=1654919r1=1654918r2=1654919view=diff == --- hive/branches/branch-1.0/common/pom.xml (original) +++ hive/branches/branch-1.0/common/pom.xml Tue Jan 27 00:05:10 2015 @@ -19,7 +19,7 @@ parent groupIdorg.apache.hive/groupId artifactIdhive/artifactId -version1.0/version +version0.14.0/version
svn commit: r1659653 - /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java
Author: vikram
Date: Fri Feb 13 19:21:59 2015
New Revision: 1659653
URL: http://svn.apache.org/r1659653
Log:
HIVE-6069: Improve error message in GenericUDFRound (Alexander Pivovarov via
Vikram Dixit)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java?rev=1659653r1=1659652r2=1659653view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java
Fri Feb 13 19:21:59 2015
@@ -22,6 +22,7 @@ import org.apache.hadoop.hive.common.typ
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.FuncRoundWithNumDigitsDecimalToDecimal;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.RoundWithNumDigitsDoubleToDouble;
@@ -83,38 +84,42 @@ public class GenericUDFRound extends Gen
ROUND requires one or two argument, got + arguments.length);
}
-inputOI = (PrimitiveObjectInspector) arguments[0];
-if (inputOI.getCategory() != Category.PRIMITIVE) {
- throw new UDFArgumentException(
- ROUND input only takes primitive types, got +
inputOI.getTypeName());
+if (arguments[0].getCategory() != Category.PRIMITIVE) {
+ throw new UDFArgumentTypeException(0,
+ ROUND input only takes primitive types, got +
arguments[0].getTypeName());
}
+inputOI = (PrimitiveObjectInspector) arguments[0];
if (arguments.length == 2) {
+ if (arguments[1].getCategory() != Category.PRIMITIVE) {
+throw new UDFArgumentTypeException(1,
+ROUND second argument only takes primitive types, got +
arguments[1].getTypeName());
+ }
PrimitiveObjectInspector scaleOI = (PrimitiveObjectInspector)
arguments[1];
switch (scaleOI.getPrimitiveCategory()) {
case VOID:
break;
case BYTE:
if (!(scaleOI instanceof WritableConstantByteObjectInspector)) {
- throw new UDFArgumentException(ROUND second argument only takes
constant);
+ throw new UDFArgumentTypeException(1, ROUND second argument only
takes constant);
}
scale =
((WritableConstantByteObjectInspector)scaleOI).getWritableConstantValue().get();
break;
case SHORT:
if (!(scaleOI instanceof WritableConstantShortObjectInspector)) {
- throw new UDFArgumentException(ROUND second argument only takes
constant);
+ throw new UDFArgumentTypeException(1, ROUND second argument only
takes constant);
}
scale =
((WritableConstantShortObjectInspector)scaleOI).getWritableConstantValue().get();
break;
case INT:
if (!(scaleOI instanceof WritableConstantIntObjectInspector)) {
- throw new UDFArgumentException(ROUND second argument only takes
constant);
+ throw new UDFArgumentTypeException(1, ROUND second argument only
takes constant);
}
scale =
((WritableConstantIntObjectInspector)scaleOI).getWritableConstantValue().get();
break;
case LONG:
if (!(scaleOI instanceof WritableConstantLongObjectInspector)) {
- throw new UDFArgumentException(ROUND second argument only takes
constant);
+ throw new UDFArgumentTypeException(1, ROUND second argument only
takes constant);
}
long l =
((WritableConstantLongObjectInspector)scaleOI).getWritableConstantValue().get();
if (l Integer.MIN_VALUE || l Integer.MAX_VALUE) {
@@ -123,7 +128,7 @@ public class GenericUDFRound extends Gen
scale = (int)l;
break;
default:
-throw new UDFArgumentException(ROUND second argument only takes
integer constant);
+throw new UDFArgumentTypeException(1, ROUND second argument only
takes integer constant);
}
}
@@ -151,8 +156,9 @@ public class GenericUDFRound extends Gen
converterFromString = ObjectInspectorConverters.getConverter(inputOI,
outputOI);
break;
default:
- throw new UDFArgumentException(Only numeric data types are allowed for
ROUND function. Got +
- inputType.name());
+ throw new UDFArgumentTypeException(0,
+ Only numeric or string group data types are allowed for ROUND
function. Got
+ + inputType.name());
}
return outputOI;
@@ -240,8 +246,9 @@ public class GenericUDFRound
svn commit: r1665378 - in /hive/branches/branch-1.0: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/ ql
Author: vikram
Date: Mon Mar 9 22:57:19 2015
New Revision: 1665378
URL: http://svn.apache.org/r1665378
Log:
HIVE-9886: Hive on tez: NPE when converting join to SMB in sub-query (Vikram
Dixit K, reviewed by Gunther Hagleitner)
Added:
hive/branches/branch-1.0/ql/src/test/queries/clientpositive/tez_join.q
hive/branches/branch-1.0/ql/src/test/results/clientpositive/tez/tez_join.q.out
Modified:
hive/branches/branch-1.0/itests/src/test/resources/testconfiguration.properties
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
Modified:
hive/branches/branch-1.0/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/branches/branch-1.0/itests/src/test/resources/testconfiguration.properties?rev=1665378r1=1665377r2=1665378view=diff
==
---
hive/branches/branch-1.0/itests/src/test/resources/testconfiguration.properties
(original)
+++
hive/branches/branch-1.0/itests/src/test/resources/testconfiguration.properties
Mon Mar 9 22:57:19 2015
@@ -283,7 +283,8 @@ minitez.query.files=bucket_map_join_tez1
tez_smb_main.q,\
tez_smb_1.q,\
vectorized_dynamic_partition_pruning.q,\
- tez_multi_union.q
+ tez_multi_union.q,\
+ tez_join.q
beeline.positive.exclude=add_part_exist.q,\
alter1.q,\
Modified:
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1665378r1=1665377r2=1665378view=diff
==
---
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
(original)
+++
hive/branches/branch-1.0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Mon Mar 9 22:57:19 2015
@@ -112,7 +112,7 @@ public class ConvertJoinMapJoin implemen
}
if (parentOp instanceof ReduceSinkOperator) {
- ReduceSinkOperator rs = (ReduceSinkOperator)parentOp;
+ ReduceSinkOperator rs = (ReduceSinkOperator) parentOp;
estimatedBuckets = (estimatedBuckets
rs.getConf().getNumReducers()) ?
rs.getConf().getNumReducers() : estimatedBuckets;
}
@@ -134,10 +134,10 @@ public class ConvertJoinMapJoin implemen
if (retval == null) {
return retval;
} else {
- // only case is full outer join with SMB enabled which is not
possible. Convert to regular
- // join.
- convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
- return null;
+// only case is full outer join with SMB enabled which is not
possible. Convert to regular
+// join.
+convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
+return null;
}
}
@@ -161,8 +161,10 @@ public class ConvertJoinMapJoin implemen
}
MapJoinOperator mapJoinOp = convertJoinMapJoin(joinOp, context,
mapJoinConversionPos);
-// map join operator by default has no bucket cols
-mapJoinOp.setOpTraits(new OpTraits(null, -1, null));
+// map join operator by default has no bucket cols and num of reduce sinks
+// reduced by 1
+mapJoinOp
+.setOpTraits(new OpTraits(null, -1, null,
joinOp.getOpTraits().getNumReduceSinks()));
mapJoinOp.setStatistics(joinOp.getStatistics());
// propagate this change till the next RS
for (Operator? extends OperatorDesc childOp :
mapJoinOp.getChildOperators()) {
@@ -176,7 +178,8 @@ public class ConvertJoinMapJoin implemen
TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException {
// we cannot convert to bucket map join, we cannot convert to
// map join either based on the size. Check if we can convert to SMB join.
-if (context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) ==
false) {
+if ((context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN)
== false)
+|| (joinOp.getOpTraits().getNumReduceSinks() = 2)) {
convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
return null;
}
@@ -221,7 +224,7 @@ public class ConvertJoinMapJoin implemen
convertJoinSMBJoin(joinOp, context, pos, 0, false, false);
}
return null;
-}
+ }
// replaces the join operator with a new CommonJoinOperator, removes the
// parent reduce sinks
@@ -249,9 +252,9 @@ public class ConvertJoinMapJoin implemen
CommonMergeJoinOperator mergeJoinOp =
(CommonMergeJoinOperator) OperatorFactory.get(new
CommonMergeJoinDesc(numBuckets,
isSubQuery, mapJoinConversionPos
svn commit: r1655889 - /hive/branches/branch-1.0/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
Author: vikram
Date: Fri Jan 30 00:01:30 2015
New Revision: 1655889
URL: http://svn.apache.org/r1655889
Log:
HIVE-9436: RetryingMetaStoreClient does not retry JDOExceptions (Sushanth
Sowmyan via Vikram Dixit)
Modified:
hive/branches/branch-1.0/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
Modified:
hive/branches/branch-1.0/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-1.0/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java?rev=1655889r1=1655888r2=1655889view=diff
==
---
hive/branches/branch-1.0/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
(original)
+++
hive/branches/branch-1.0/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
Fri Jan 30 00:01:30 2015
@@ -97,7 +97,7 @@ public class RetryingMetaStoreClient imp
(e.getCause() instanceof TTransportException)) {
caughtException = (TException) e.getCause();
} else if ((e.getCause() instanceof MetaException)
-e.getCause().getMessage().matches(JDO[a-zA-Z]*Exception)) {
+
e.getCause().getMessage().matches((?s).*JDO[a-zA-Z]*Exception.*)) {
caughtException = (MetaException) e.getCause();
} else {
throw e.getCause();
svn commit: r1655898 - /hive/tags/release-1.0.0-rc2/
Author: vikram Date: Fri Jan 30 00:44:44 2015 New Revision: 1655898 URL: http://svn.apache.org/r1655898 Log: Creating the tag 1.0.0 Added: hive/tags/release-1.0.0-rc2/ (props changed) - copied from r1655897, hive/branches/branch-1.0/ Propchange: hive/tags/release-1.0.0-rc2/ -- reviewboard:url = https://reviews.apache.org Propchange: hive/tags/release-1.0.0-rc2/ -- --- svn:ignore (added) +++ svn:ignore Fri Jan 30 00:44:44 2015 @@ -0,0 +1,11 @@ +datanucleus.log +eclipse-build +target +.arc_jira_lib +.classpath* +.externalToolBuilders +.project +.settings +*.launch +*~ +metastore_db Propchange: hive/tags/release-1.0.0-rc2/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Fri Jan 30 00:44:44 2015 @@ -0,0 +1,5 @@ +/hive/branches/branch-0.11:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184 +/hive/branches/cbo:1605012-1627125 +/hive/branches/tez:1494760-1622766 +/hive/branches/vectorization:1466908-1527856 +/hive/trunk:1627830,1629752,1630512,1635373,1637170,1639249,1655460
svn commit: r1655888 - /hive/branches/branch-1.0/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
Author: vikram Date: Thu Jan 29 23:51:13 2015 New Revision: 1655888 URL: http://svn.apache.org/r1655888 Log: HIVE-8807: Obsolete default values in webhcat-default.xml (Eugene Koifman via Vikram Dixit) Modified: hive/branches/branch-1.0/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Modified: hive/branches/branch-1.0/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1655888r1=1655887r2=1655888view=diff == --- hive/branches/branch-1.0/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original) +++ hive/branches/branch-1.0/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Thu Jan 29 23:51:13 2015 @@ -87,7 +87,7 @@ property nametempleton.pig.path/name -valuepig-0.11.1.tar.gz/pig-0.11.1/bin/pig/value +valuepig-0.12.2.tar.gz/pig-0.12.2/bin/pig/value descriptionThe path to the Pig executable./description /property @@ -105,13 +105,13 @@ property nametempleton.hive.path/name -valuehive-0.11.0.tar.gz/hive-0.11.0/bin/hive/value +valueapache-hive-1.0.0-bin.tar.gz/apache-hive-1.0-bin/bin/hive/value descriptionThe path to the Hive executable./description /property property nametempleton.hive.home/name -valuehive-0.14.0-SNAPSHOT-bin.tar.gz/hive-0.14.0-SNAPSHOT-bin/value +valueapache-hive-1.0.0-bin.tar.gz/apache-hive-1.0-bin/value description The path to the Hive home within the tar. This is needed if Hive is not installed on all nodes in the cluster and needs to be shipped to the target node in the cluster to execute Pig @@ -120,7 +120,7 @@ /property property nametempleton.hcat.home/name - valuehive-0.14.0-SNAPSHOT-bin.tar.gz/hive-0.14.0-SNAPSHOT-bin/hcatalog/value +valueapache-hive-1.0.0-bin.tar.gz/apache-hive-1.0-bin/hcatalog/value description The path to the HCat home within the tar. This is needed if Hive is not installed on all nodes in the cluster and needs to be shipped to the target node in the cluster to execute Pig
svn commit: r1655896 - in /hive/branches/branch-1.0: NOTICE README.txt RELEASE_NOTES.txt
Author: vikram Date: Fri Jan 30 00:40:46 2015 New Revision: 1655896 URL: http://svn.apache.org/r1655896 Log: Branch-1.0: Update notice, readme and release notes. Modified: hive/branches/branch-1.0/NOTICE hive/branches/branch-1.0/README.txt hive/branches/branch-1.0/RELEASE_NOTES.txt Modified: hive/branches/branch-1.0/NOTICE URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/NOTICE?rev=1655896r1=1655895r2=1655896view=diff == --- hive/branches/branch-1.0/NOTICE (original) +++ hive/branches/branch-1.0/NOTICE Fri Jan 30 00:40:46 2015 @@ -1,5 +1,5 @@ Apache Hive -Copyright 2008-2014 The Apache Software Foundation +Copyright 2008-2015 The Apache Software Foundation This product includes software developed by The Apache Software Foundation (http://www.apache.org/). Modified: hive/branches/branch-1.0/README.txt URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/README.txt?rev=1655896r1=1655895r2=1655896view=diff == --- hive/branches/branch-1.0/README.txt (original) +++ hive/branches/branch-1.0/README.txt Fri Jan 30 00:40:46 2015 @@ -1,4 +1,4 @@ -Apache Hive (TM) @VERSION@ +Apache Hive (TM) 1.0.0 == The Apache Hive (TM) data warehouse software facilitates querying and Modified: hive/branches/branch-1.0/RELEASE_NOTES.txt URL: http://svn.apache.org/viewvc/hive/branches/branch-1.0/RELEASE_NOTES.txt?rev=1655896r1=1655895r2=1655896view=diff == --- hive/branches/branch-1.0/RELEASE_NOTES.txt (original) +++ hive/branches/branch-1.0/RELEASE_NOTES.txt Fri Jan 30 00:40:46 2015 @@ -15,6 +15,7 @@ Release Notes - Hive - Version 1.0.0 * [HIVE-8739] - handle Derby and Oracle errors with joins and filters in Direct SQL in a invalid-DB-specific path * [HIVE-8784] - Querying partition does not work with JDO enabled against PostgreSQL * [HIVE-8805] - CBO skipped due to SemanticException: Line 0:-1 Both left and right aliases encountered in JOIN 'avg_cs_ext_discount_amt' +* [HIVE-8807] - Obsolete default values in webhcat-default.xml * [HIVE-8811] - Dynamic partition pruning can result in NPE during query compilation * [HIVE-8827] - Remove SSLv2Hello from list of disabled protocols * [HIVE-8830] - hcatalog process don't exit because of non daemon thread @@ -49,11 +50,15 @@ Release Notes - Hive - Version 1.0.0 * [HIVE-9234] - HiveServer2 leaks FileSystem objects in FileSystem.CACHE * [HIVE-9249] - java.lang.ClassCastException: org.apache.hadoop.hive.serde2.io.HiveVarcharWritable cannot be cast to org.apache.hadoop.hive.common.type.HiveVarchar when joining tables * [HIVE-9278] - Cached expression feature broken in one case +* [HIVE-9317] - move Microsoft copyright to NOTICE file * [HIVE-9359] - Export of a large table causes OOM in Metastore and Client * [HIVE-9361] - Intermittent NPE in SessionHiveMetaStoreClient.alterTempTable * [HIVE-9390] - Enhance retry logic wrt DB access in TxnHandler * [HIVE-9401] - Backport: Fastpath for limited fetches from unpartitioned tables * [HIVE-9404] - NPE in org.apache.hadoop.hive.metastore.txn.TxnHandler.determineDatabaseProduct() +* [HIVE-9436] - RetryingMetaStoreClient does not retry JDOExceptions +* [HIVE-9473] - sql std auth should disallow built-in udfs that allow any java methods to be called +* [HIVE-9514] - schematool is broken in hive 1.0.0 ** Improvement
svn commit: r1670971 - in /hive/branches/llap/llap-server/src: java/org/apache/hadoop/hive/llap/cli/ java/org/apache/hadoop/hive/llap/daemon/impl/ java/org/apache/hadoop/hive/llap/daemon/registry/impl
Author: vikram
Date: Thu Apr 2 21:17:25 2015
New Revision: 1670971
URL: http://svn.apache.org/r1670971
Log:
Revert HIVE-10201. Missed files.
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
hive/branches/llap/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java?rev=1670971r1=1670970r2=1670971view=diff
==
---
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
(original)
+++
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
Thu Apr 2 21:17:25 2015
@@ -32,7 +32,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.CompressionUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.cli.LlapOptionsProcessor.LlapOptions;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
+import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration;
import org.apache.hadoop.hive.llap.io.api.impl.LlapInputFormat;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
@@ -121,7 +121,7 @@ public class LlapServiceDriver {
if (options.getName() != null) {
// update service registry configs - caveat: this has nothing to do with
the actual settings as read by the AM
// if needed, use --hiveconf llap.daemon.service.hosts=@llap0 to
dynamically switch between instances
- conf.set(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS, @ +
options.getName());
+ conf.set(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS, @ +
options.getName());
}
URL logger = conf.getResource(llap-daemon-log4j.properties);
@@ -188,17 +188,17 @@ public class LlapServiceDriver {
configs.put(HiveConf.ConfVars.LLAP_ORC_CACHE_ALLOCATE_DIRECT.varname,
HiveConf.getBoolVar(conf,
HiveConf.ConfVars.LLAP_ORC_CACHE_ALLOCATE_DIRECT));
-configs.put(LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
conf.getInt(
-LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
-LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT));
-
-configs.put(LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE, conf.getInt(
-LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE,
-LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT));
-
-configs.put(LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS, conf.getInt(
-LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS,
-LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT));
+configs.put(LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
conf.getInt(
+LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
+LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT));
+
+configs.put(LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE,
conf.getInt(
+LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE,
+LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT));
+
+configs.put(LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS, conf.getInt(
+LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS,
+LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT));
configs.put(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1));
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java?rev=1670971r1=1670970r2=1670971view=diff
==
---
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
(original)
+++
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
Thu Apr 2 21:17:25 2015
svn commit: r1670961 - in /hive/branches/llap/llap-server/src: java/org/apache/hadoop/hive/llap/cli/ java/org/apache/hadoop/hive/llap/daemon/impl/ java/org/apache/hadoop/hive/llap/daemon/registry/impl
Author: vikram
Date: Thu Apr 2 20:38:53 2015
New Revision: 1670961
URL: http://svn.apache.org/r1670961
Log:
HIVE-10201: Hive LLAP needs refactoring of the configuration class (Vikram
Dixit K, reviewed by Siddharth Seth)
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
hive/branches/llap/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java?rev=1670961r1=1670960r2=1670961view=diff
==
---
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
(original)
+++
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
Thu Apr 2 20:38:53 2015
@@ -32,7 +32,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.CompressionUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.cli.LlapOptionsProcessor.LlapOptions;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration;
+import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
import org.apache.hadoop.hive.llap.io.api.impl.LlapInputFormat;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
@@ -121,7 +121,7 @@ public class LlapServiceDriver {
if (options.getName() != null) {
// update service registry configs - caveat: this has nothing to do with
the actual settings as read by the AM
// if needed, use --hiveconf llap.daemon.service.hosts=@llap0 to
dynamically switch between instances
- conf.set(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS, @ +
options.getName());
+ conf.set(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS, @ +
options.getName());
}
URL logger = conf.getResource(llap-daemon-log4j.properties);
@@ -188,17 +188,17 @@ public class LlapServiceDriver {
configs.put(HiveConf.ConfVars.LLAP_ORC_CACHE_ALLOCATE_DIRECT.varname,
HiveConf.getBoolVar(conf,
HiveConf.ConfVars.LLAP_ORC_CACHE_ALLOCATE_DIRECT));
-configs.put(LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
conf.getInt(
-LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
-LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT));
-
-configs.put(LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE,
conf.getInt(
-LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE,
-LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT));
-
-configs.put(LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS, conf.getInt(
-LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS,
-LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT));
+configs.put(LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
conf.getInt(
+LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
+LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT));
+
+configs.put(LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE, conf.getInt(
+LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE,
+LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT));
+
+configs.put(LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS, conf.getInt(
+LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS,
+LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT));
configs.put(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1));
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java?rev=1670961r1=1670960r2=1670961view=diff
==
---
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
(original)
+++
hive/branches/llap/llap-server/src/java/org
svn commit: r1670973 - in /hive/branches/llap: llap-client/src/java/org/apache/hadoop/hive/llap/configuration/ llap-server/src/java/org/apache/hadoop/hive/llap/cli/ llap-server/src/java/org/apache/had
Author: vikram
Date: Thu Apr 2 21:18:45 2015
New Revision: 1670973
URL: http://svn.apache.org/r1670973
Log:
HIVE-10201: Hive LLAP needs refactoring of the configuration class (Vikram
Dixit K, reviewed by Siddharth Seth)
Added:
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
Removed:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
hive/branches/llap/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
Added:
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java?rev=1670973view=auto
==
---
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
(added)
+++
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
Thu Apr 2 21:18:45 2015
@@ -0,0 +1,78 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the License);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.configuration;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class LlapConfiguration extends Configuration {
+
+ public LlapConfiguration(Configuration conf) {
+super(conf);
+addResource(LLAP_DAEMON_SITE);
+ }
+
+ public LlapConfiguration() {
+super(false);
+addResource(LLAP_DAEMON_SITE);
+ }
+
+
+ public static final String LLAP_DAEMON_PREFIX = llap.daemon.;
+ private static final String LLAP_DAEMON_SITE = llap-daemon-site.xml;
+
+
+
+ public static final String LLAP_DAEMON_RPC_NUM_HANDLERS = LLAP_DAEMON_PREFIX
+ rpc.num.handlers;
+ public static final int LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT = 5;
+
+ public static final String LLAP_DAEMON_WORK_DIRS = LLAP_DAEMON_PREFIX +
work.dirs;
+
+ public static final String LLAP_DAEMON_YARN_SHUFFLE_PORT =
LLAP_DAEMON_PREFIX + yarn.shuffle.port;
+ public static final int LLAP_DAEMON_YARN_SHUFFLE_PORT_DEFAULT = 15551;
+
+ public static final String LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED =
LLAP_DAEMON_PREFIX + shuffle.dir-watcher.enabled;
+ public static final boolean LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT
= false;
+
+
+ // Section for configs used in AM and executors
+ public static final String LLAP_DAEMON_NUM_EXECUTORS = LLAP_DAEMON_PREFIX +
num.executors;
+ public static final int LLAP_DAEMON_NUM_EXECUTORS_DEFAULT = 4;
+
+ public static final String LLAP_DAEMON_RPC_PORT = LLAP_DAEMON_PREFIX +
rpc.port;
+ public static final int LLAP_DAEMON_RPC_PORT_DEFAULT = 15001;
+
+ public static final String LLAP_DAEMON_MEMORY_PER_INSTANCE_MB =
LLAP_DAEMON_PREFIX + memory.per.instance.mb;
+ public static final int LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT = 4096;
+
+ public static final String LLAP_DAEMON_VCPUS_PER_INSTANCE =
LLAP_DAEMON_PREFIX + vcpus.per.instance;
+ public static final int LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT = 4;
+
+
+ // Section for configs used in the AM //
+ public static final String LLAP_DAEMON_SERVICE_HOSTS = LLAP_DAEMON_PREFIX +
service.hosts;
+
+ public static final String LLAP_DAEMON_COMMUNICATOR_NUM_THREADS =
LLAP_DAEMON_PREFIX + communicator.num.threads;
+ public static final int LLAP_DAEMON_COMMUNICATOR_NUM_THREADS_DEFAULT = 5;
+
+ /**
+ * Time after which a previously disabled node will be re-enabled for
scheduling. This may be
+ * modified by an exponential
svn commit: r1675017 - /hive/trunk/ql/src/test/results/clientpositive/union_view.q.out
Author: vikram Date: Tue Apr 21 01:05:46 2015 New Revision: 1675017 URL: http://svn.apache.org/r1675017 Log: HIVE-10352: Investigate the failure of union_view.q (Vikram Dixit, reviewed by Thejas Nair) Modified: hive/trunk/ql/src/test/results/clientpositive/union_view.q.out Modified: hive/trunk/ql/src/test/results/clientpositive/union_view.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_view.q.out?rev=1675017r1=1675016r2=1675017view=diff == --- hive/trunk/ql/src/test/results/clientpositive/union_view.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/union_view.q.out Tue Apr 21 01:05:46 2015 @@ -373,6 +373,54 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan +alias: src_union_2 +filterExpr: ((key = 86) and (ds = '1')) (type: boolean) +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Filter Operator + predicate: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator +expressions: value (type: string) +outputColumnNames: _col1 +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Union + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: 86 (type: int), _col1 (type: string), '1' (type: string) +outputColumnNames: _col0, _col1, _col2 +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan +alias: src_union_3 +filterExpr: ((key = 86) and (ds = '1')) (type: boolean) +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Filter Operator + predicate: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator +expressions: value (type: string) +outputColumnNames: _col1 +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Union + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: 86 (type: int), _col1 (type: string), '1' (type: string) +outputColumnNames: _col0, _col1, _col2 +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -415,6 +463,30 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan +alias: src_union_1 +filterExpr: ((key = 86) and (ds = '2')) (type: boolean) +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Filter Operator + predicate: ((key = 86) and (ds = '2')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator +expressions: value (type: string) +outputColumnNames: _col1 +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator
[2/5] hive git commit: HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via Vikram Dixit)
http://git-wip-us.apache.org/repos/asf/hive/blob/c37d18fb/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out new file mode 100644 index 000..4ad143c --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out @@ -0,0 +1,1417 @@ +PREHOOK: query: -- Hybrid Grace Hash Join +-- Test n-way join +SELECT 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +POSTHOOK: query: -- Hybrid Grace Hash Join +-- Test n-way join +SELECT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +1 +PREHOOK: query: -- 3-way mapjoin (1 big table, 2 small tables) +SELECT 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +POSTHOOK: query: -- 3-way mapjoin (1 big table, 2 small tables) +SELECT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +1 +PREHOOK: query: EXPLAIN +SELECT COUNT(*) +FROM src1 x JOIN srcpart z ON (x.key = z.key) +JOIN src y ON (y.key = x.key) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT COUNT(*) +FROM src1 x JOIN srcpart z ON (x.key = z.key) +JOIN src y ON (y.key = x.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Tez + Edges: +Map 2 - Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +Reducer 3 - Map 2 (SIMPLE_EDGE) + A masked pattern was here + Vertices: +Map 1 +Map Operator Tree: +TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE +Map 2 +Map Operator Tree: +TableScan + alias: z + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE +Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: +0 key (type: string) +1 key (type: string) +2 key (type: string) + input vertices: +0 Map 1 +2 Map 4 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator +aggregations: count() +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) +Map 4 +Map Operator Tree: +TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Reducer 3 +Reduce Operator Tree: + Group By Operator +aggregations: count(VALUE._col0) +mode: mergepartial +outputColumnNames: _col0 +
[3/5] hive git commit: HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via Vikram Dixit)
http://git-wip-us.apache.org/repos/asf/hive/blob/c37d18fb/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out new file mode 100644 index 000..4b39b2c --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out @@ -0,0 +1,1587 @@ +PREHOOK: query: -- Hybrid Grace Hash Join +-- Test basic functionalities: +-- 1. Various cases when hash partitions spill +-- 2. Partitioned table spilling +-- 3. Vectorization + +SELECT 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +POSTHOOK: query: -- Hybrid Grace Hash Join +-- Test basic functionalities: +-- 1. Various cases when hash partitions spill +-- 2. Partitioned table spilling +-- 3. Vectorization + +SELECT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +1 +PREHOOK: query: -- Base result for inner join +explain +select count(*) from +(select c.ctinyint + from alltypesorc c + inner join alltypesorc cd + on cd.cint = c.cint + where c.cint 20) t1 +PREHOOK: type: QUERY +POSTHOOK: query: -- Base result for inner join +explain +select count(*) from +(select c.ctinyint + from alltypesorc c + inner join alltypesorc cd + on cd.cint = c.cint + where c.cint 20) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Tez + Edges: +Map 1 - Map 3 (BROADCAST_EDGE) +Reducer 2 - Map 1 (SIMPLE_EDGE) + A masked pattern was here + Vertices: +Map 1 +Map Operator Tree: +TableScan + alias: c + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (cint 20) (type: boolean) +Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE + Map Join Operator +condition map: + Inner Join 0 to 1 +keys: + 0 _col0 (type: int) + 1 _col0 (type: int) +input vertices: + 1 Map 3 +Statistics: Num rows: 4505 Data size: 968719 Basic stats: COMPLETE Column stats: NONE +Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +sort order: +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: bigint) +Map 3 +Map Operator Tree: +TableScan + alias: c + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (cint 20) (type: boolean) +Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col0 (type: int) +sort order: + +Map-reduce partition columns: _col0 (type: int) +Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE +Reducer 2 +Reduce Operator Tree: + Group By Operator +aggregations: count(VALUE._col0) +mode: mergepartial +outputColumnNames: _col0 +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table:
[1/5] hive git commit: HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via Vikram Dixit)
Repository: hive Updated Branches: refs/heads/branch-1.2 343486b65 - c37d18fba http://git-wip-us.apache.org/repos/asf/hive/blob/c37d18fb/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out b/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out deleted file mode 100644 index 5d5b850..000 --- a/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out +++ /dev/null @@ -1,1566 +0,0 @@ -PREHOOK: query: -- Base result for inner join -explain -select count(*) from -(select c.ctinyint - from alltypesorc c - inner join alltypesorc cd - on cd.cint = c.cint - where c.cint 20) t1 -PREHOOK: type: QUERY -POSTHOOK: query: -- Base result for inner join -explain -select count(*) from -(select c.ctinyint - from alltypesorc c - inner join alltypesorc cd - on cd.cint = c.cint - where c.cint 20) t1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 -Tez - Edges: -Map 1 - Map 3 (BROADCAST_EDGE) -Reducer 2 - Map 1 (SIMPLE_EDGE) - A masked pattern was here - Vertices: -Map 1 -Map Operator Tree: -TableScan - alias: c - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (cint 20) (type: boolean) -Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE - Map Join Operator -condition map: - Inner Join 0 to 1 -keys: - 0 _col0 (type: int) - 1 _col0 (type: int) -input vertices: - 1 Map 3 -Statistics: Num rows: 4505 Data size: 968719 Basic stats: COMPLETE Column stats: NONE -Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -sort order: -Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: bigint) -Map 3 -Map Operator Tree: -TableScan - alias: c - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (cint 20) (type: boolean) -Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col0 (type: int) -sort order: + -Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE -Reducer 2 -Reduce Operator Tree: - Group By Operator -aggregations: count(VALUE._col0) -mode: mergepartial -outputColumnNames: _col0 -Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 -Fetch Operator - limit: -1 - Processor Tree: -ListSink - -PREHOOK: query: select count(*) from -(select c.ctinyint - from alltypesorc c - inner join alltypesorc cd - on cd.cint = c.cint - where c.cint 20) t1 -PREHOOK: type: QUERY
[1/5] hive git commit: HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via Vikram Dixit)
Repository: hive Updated Branches: refs/heads/master 6db33a9d1 - c72d073c1 http://git-wip-us.apache.org/repos/asf/hive/blob/c72d073c/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out b/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out deleted file mode 100644 index 5d5b850..000 --- a/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out +++ /dev/null @@ -1,1566 +0,0 @@ -PREHOOK: query: -- Base result for inner join -explain -select count(*) from -(select c.ctinyint - from alltypesorc c - inner join alltypesorc cd - on cd.cint = c.cint - where c.cint 20) t1 -PREHOOK: type: QUERY -POSTHOOK: query: -- Base result for inner join -explain -select count(*) from -(select c.ctinyint - from alltypesorc c - inner join alltypesorc cd - on cd.cint = c.cint - where c.cint 20) t1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 -Tez - Edges: -Map 1 - Map 3 (BROADCAST_EDGE) -Reducer 2 - Map 1 (SIMPLE_EDGE) - A masked pattern was here - Vertices: -Map 1 -Map Operator Tree: -TableScan - alias: c - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (cint 20) (type: boolean) -Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE - Map Join Operator -condition map: - Inner Join 0 to 1 -keys: - 0 _col0 (type: int) - 1 _col0 (type: int) -input vertices: - 1 Map 3 -Statistics: Num rows: 4505 Data size: 968719 Basic stats: COMPLETE Column stats: NONE -Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -sort order: -Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: bigint) -Map 3 -Map Operator Tree: -TableScan - alias: c - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (cint 20) (type: boolean) -Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col0 (type: int) -sort order: + -Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE -Reducer 2 -Reduce Operator Tree: - Group By Operator -aggregations: count(VALUE._col0) -mode: mergepartial -outputColumnNames: _col0 -Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 -Fetch Operator - limit: -1 - Processor Tree: -ListSink - -PREHOOK: query: select count(*) from -(select c.ctinyint - from alltypesorc c - inner join alltypesorc cd - on cd.cint = c.cint - where c.cint 20) t1 -PREHOOK: type: QUERY
[5/5] hive git commit: HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via Vikram Dixit)
HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via
Vikram Dixit)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c72d073c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c72d073c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c72d073c
Branch: refs/heads/master
Commit: c72d073c1fe2a07c80120647bb2170fb7e50d168
Parents: 6db33a9
Author: vikram vik...@hortonworks.com
Authored: Fri May 1 10:54:21 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Fri May 1 11:07:06 2015 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java |4 +
.../test/resources/testconfiguration.properties |3 +-
.../hadoop/hive/ql/exec/HashTableLoader.java|4 +-
.../hadoop/hive/ql/exec/MapJoinOperator.java| 169 +-
.../hadoop/hive/ql/exec/mr/HashTableLoader.java |2 +-
.../persistence/BytesBytesMultiHashMap.java |1 +
.../exec/persistence/HybridHashTableConf.java | 86 +
.../persistence/HybridHashTableContainer.java | 213 ++-
.../ql/exec/persistence/KeyValueContainer.java | 31 +-
.../ql/exec/persistence/ObjectContainer.java| 31 +-
.../hive/ql/exec/spark/HashTableLoader.java |2 +-
.../hive/ql/exec/tez/HashTableLoader.java | 60 +-
.../ql/exec/vector/VectorMapJoinOperator.java |4 +-
.../mapjoin/VectorMapJoinCommonOperator.java|4 -
.../VectorMapJoinGenerateResultOperator.java| 25 +-
.../fast/VectorMapJoinFastBytesHashMap.java |4 +-
.../VectorMapJoinFastBytesHashMultiSet.java |4 +-
.../fast/VectorMapJoinFastBytesHashSet.java |4 +-
.../fast/VectorMapJoinFastBytesHashTable.java |4 +-
.../mapjoin/fast/VectorMapJoinFastHashMap.java |4 +-
.../fast/VectorMapJoinFastHashMultiSet.java |4 +-
.../mapjoin/fast/VectorMapJoinFastHashSet.java |4 +-
.../fast/VectorMapJoinFastHashTable.java|4 +-
.../fast/VectorMapJoinFastHashTableLoader.java |4 +-
.../fast/VectorMapJoinFastLongHashMap.java |4 +-
.../fast/VectorMapJoinFastLongHashMultiSet.java |4 +-
.../fast/VectorMapJoinFastLongHashSet.java |4 +-
.../fast/VectorMapJoinFastLongHashTable.java|4 +-
.../fast/VectorMapJoinFastMultiKeyHashMap.java |6 +-
.../VectorMapJoinFastMultiKeyHashMultiSet.java |4 +-
.../fast/VectorMapJoinFastMultiKeyHashSet.java |4 +-
.../fast/VectorMapJoinFastStringHashMap.java|4 +-
.../VectorMapJoinFastStringHashMultiSet.java|4 +-
.../fast/VectorMapJoinFastStringHashSet.java|4 +-
.../fast/VectorMapJoinFastTableContainer.java | 23 +-
.../apache/hadoop/hive/ql/plan/MapJoinDesc.java |2 +-
.../fast/TestVectorMapJoinFastLongHashMap.java | 14 +-
.../TestVectorMapJoinFastMultiKeyHashMap.java | 14 +-
.../clientpositive/auto_sortmerge_join_13.q |2 +
.../clientpositive/hybridgrace_hashjoin_1.q | 258 +++
.../clientpositive/hybridgrace_hashjoin_2.q | 152 ++
.../queries/clientpositive/hybridhashjoin.q | 250 ---
.../test/queries/clientpositive/tez_join_hash.q |2 +
.../test/queries/clientpositive/tez_smb_main.q |6 +
.../tez/hybridgrace_hashjoin_1.q.out| 1587 ++
.../tez/hybridgrace_hashjoin_2.q.out| 1417
.../clientpositive/tez/hybridhashjoin.q.out | 1566 -
47 files changed, 3924 insertions(+), 2086 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/c72d073c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 72e4ff2..95e3d04 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -764,6 +764,10 @@ public class HiveConf extends Configuration {
HIVEHYBRIDGRACEHASHJOINMEMCHECKFREQ(hive.mapjoin.hybridgrace.memcheckfrequency,
1024, For +
hybrid grace hash join, how often (how many rows apart) we check if
memory is full. +
This number should be power of 2.),
+HIVEHYBRIDGRACEHASHJOINMINWBSIZE(hive.mapjoin.hybridgrace.minwbsize,
524288, For hybrid grace +
+ hash join, the minimum write buffer size used by optimized
hashtable. Default is 512 KB.),
+
HIVEHYBRIDGRACEHASHJOINMINNUMPARTITIONS(hive.mapjoin.hybridgrace.minnumpartitions,
16, For +
+ hybrid grace hash join, the minimum number of partitions to
create.),
HIVEHASHTABLEWBSIZE(hive.mapjoin.optimized.hashtable.wbsize, 10 * 1024 *
1024,
Optimized hashtable (see hive.mapjoin.optimized.hashtable) uses a
chain of buffers to\n +
store data
[4/5] hive git commit: HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via Vikram Dixit)
http://git-wip-us.apache.org/repos/asf/hive/blob/c72d073c/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
index 0192fb5..cee9100 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
@@ -67,7 +67,7 @@ public class MapJoinDesc extends JoinDesc implements
Serializable {
private boolean isBucketMapJoin;
// Hash table memory usage allowed; used in case of non-staged mapjoin.
- private float hashtableMemoryUsage;
+ private float hashtableMemoryUsage; // This is a percentage value between
0 and 1
protected boolean genJoinKeys = true;
private boolean isHybridHashJoin;
http://git-wip-us.apache.org/repos/asf/hive/blob/c72d073c/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java
index eb38b19..a45275b 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java
@@ -35,7 +35,7 @@ public class TestVectorMapJoinFastLongHashMap extends
CommonFastHashTable {
random = new Random(47496);
VectorMapJoinFastLongHashMap map =
-new VectorMapJoinFastLongHashMap(false, false, HashTableKeyType.LONG,
CAPACITY, LOAD_FACTOR, WB_SIZE, 0);
+new VectorMapJoinFastLongHashMap(false, false, HashTableKeyType.LONG,
CAPACITY, LOAD_FACTOR, WB_SIZE);
RandomLongStream randomLongKeyStream = new RandomLongStream(random);
RandomByteArrayStream randomByteArrayValueStream = new
RandomByteArrayStream(random);
@@ -55,7 +55,7 @@ public class TestVectorMapJoinFastLongHashMap extends
CommonFastHashTable {
public void testPutGetMultiple() throws Exception {
random = new Random(2990);
-VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap(false,
false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, 0);
+VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap(false,
false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE);
RandomLongStream randomLongKeyStream = new RandomLongStream(random);
RandomByteArrayStream randomByteArrayValueStream = new
RandomByteArrayStream(random);
@@ -77,7 +77,7 @@ public class TestVectorMapJoinFastLongHashMap extends
CommonFastHashTable {
public void testGetNonExistent() throws Exception {
random = new Random(16916);
-VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap(false,
false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, 0);
+VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap(false,
false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE);
RandomLongStream randomLongKeyStream = new RandomLongStream(random);
RandomByteArrayStream randomByteArrayValueStream = new
RandomByteArrayStream(random);
@@ -101,7 +101,7 @@ public class TestVectorMapJoinFastLongHashMap extends
CommonFastHashTable {
random = new Random(26078);
// Make sure the map does not expand; should be able to find space.
-VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap(false,
false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE, 0);
+VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap(false,
false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE);
RandomLongStream randomLongKeyStream = new RandomLongStream(random);
RandomByteArrayStream randomByteArrayValueStream = new
RandomByteArrayStream(random);
@@ -126,7 +126,7 @@ public class TestVectorMapJoinFastLongHashMap extends
CommonFastHashTable {
random = new Random(22470);
// Start with capacity 1; make sure we expand on every put.
-VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap(false,
false, HashTableKeyType.LONG, 1, 0.001f, WB_SIZE, 0);
+VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap(false,
false, HashTableKeyType.LONG, 1, 0.001f, WB_SIZE);
RandomLongStream randomLongKeyStream = new RandomLongStream(random);
RandomByteArrayStream randomByteArrayValueStream = new
RandomByteArrayStream(random);
@@ -147,7 +147,7 @@ public class TestVectorMapJoinFastLongHashMap extends
CommonFastHashTable {
random = new Random(40719);
// Use a large capacity that doesn't require expansion, yet.
-VectorMapJoinFastLongHashMap map = new
[2/5] hive git commit: HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via Vikram Dixit)
http://git-wip-us.apache.org/repos/asf/hive/blob/c72d073c/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out new file mode 100644 index 000..4ad143c --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out @@ -0,0 +1,1417 @@ +PREHOOK: query: -- Hybrid Grace Hash Join +-- Test n-way join +SELECT 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +POSTHOOK: query: -- Hybrid Grace Hash Join +-- Test n-way join +SELECT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +1 +PREHOOK: query: -- 3-way mapjoin (1 big table, 2 small tables) +SELECT 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +POSTHOOK: query: -- 3-way mapjoin (1 big table, 2 small tables) +SELECT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +1 +PREHOOK: query: EXPLAIN +SELECT COUNT(*) +FROM src1 x JOIN srcpart z ON (x.key = z.key) +JOIN src y ON (y.key = x.key) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT COUNT(*) +FROM src1 x JOIN srcpart z ON (x.key = z.key) +JOIN src y ON (y.key = x.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Tez + Edges: +Map 2 - Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +Reducer 3 - Map 2 (SIMPLE_EDGE) + A masked pattern was here + Vertices: +Map 1 +Map Operator Tree: +TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE +Map 2 +Map Operator Tree: +TableScan + alias: z + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE +Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: +0 key (type: string) +1 key (type: string) +2 key (type: string) + input vertices: +0 Map 1 +2 Map 4 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Group By Operator +aggregations: count() +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) +Map 4 +Map Operator Tree: +TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: key is not null (type: boolean) +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Reducer 3 +Reduce Operator Tree: + Group By Operator +aggregations: count(VALUE._col0) +mode: mergepartial +outputColumnNames: _col0 +
[3/5] hive git commit: HIVE-10403 - Add n-way join support for Hybrid Grace Hash Join (Wei Zheng via Vikram Dixit)
http://git-wip-us.apache.org/repos/asf/hive/blob/c72d073c/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out new file mode 100644 index 000..4b39b2c --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out @@ -0,0 +1,1587 @@ +PREHOOK: query: -- Hybrid Grace Hash Join +-- Test basic functionalities: +-- 1. Various cases when hash partitions spill +-- 2. Partitioned table spilling +-- 3. Vectorization + +SELECT 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +POSTHOOK: query: -- Hybrid Grace Hash Join +-- Test basic functionalities: +-- 1. Various cases when hash partitions spill +-- 2. Partitioned table spilling +-- 3. Vectorization + +SELECT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table + A masked pattern was here +1 +PREHOOK: query: -- Base result for inner join +explain +select count(*) from +(select c.ctinyint + from alltypesorc c + inner join alltypesorc cd + on cd.cint = c.cint + where c.cint 20) t1 +PREHOOK: type: QUERY +POSTHOOK: query: -- Base result for inner join +explain +select count(*) from +(select c.ctinyint + from alltypesorc c + inner join alltypesorc cd + on cd.cint = c.cint + where c.cint 20) t1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Tez + Edges: +Map 1 - Map 3 (BROADCAST_EDGE) +Reducer 2 - Map 1 (SIMPLE_EDGE) + A masked pattern was here + Vertices: +Map 1 +Map Operator Tree: +TableScan + alias: c + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (cint 20) (type: boolean) +Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE + Map Join Operator +condition map: + Inner Join 0 to 1 +keys: + 0 _col0 (type: int) + 1 _col0 (type: int) +input vertices: + 1 Map 3 +Statistics: Num rows: 4505 Data size: 968719 Basic stats: COMPLETE Column stats: NONE +Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +sort order: +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: bigint) +Map 3 +Map Operator Tree: +TableScan + alias: c + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (cint 20) (type: boolean) +Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col0 (type: int) +sort order: + +Map-reduce partition columns: _col0 (type: int) +Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE +Reducer 2 +Reduce Operator Tree: + Group By Operator +aggregations: count(VALUE._col0) +mode: mergepartial +outputColumnNames: _col0 +Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table:
hive git commit: HIVE-10611: Mini tez tests wait for 5 minutes before shutting down (Vikram Dixit K, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 18fb46017 - 632a30908
HIVE-10611: Mini tez tests wait for 5 minutes before shutting down (Vikram
Dixit K, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/632a3090
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/632a3090
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/632a3090
Branch: refs/heads/master
Commit: 632a30908d4680c0ee31ba04e4cfc9da3554e4f5
Parents: 18fb460
Author: vikram vik...@hortonworks.com
Authored: Wed May 6 14:28:23 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Wed May 6 14:30:14 2015 -0700
--
.../main/java/org/apache/hadoop/hive/ql/QTestUtil.java | 3 +++
.../hadoop/hive/ql/exec/tez/TezSessionState.java | 13 +++--
2 files changed, 10 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/632a3090/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
--
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index d1104b3..3e29d3c 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -430,6 +430,9 @@ public class QTestUtil {
cleanUp();
}
+if (clusterType == MiniClusterType.tez) {
+ SessionState.get().getTezSession().close(false);
+}
setup.tearDown();
if (sparkSession != null) {
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/632a3090/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 89286e5..cef3303 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -249,13 +249,14 @@ public class TezSessionState {
}
/**
- * Close a tez session. Will cleanup any tez/am related resources. After
closing a session
- * no further DAGs can be executed against it.
- * @param keepTmpDir whether or not to remove the scratch dir at the same
time.
- * @throws IOException
- * @throws TezException
+ * Close a tez session. Will cleanup any tez/am related resources. After
closing a session no
+ * further DAGs can be executed against it.
+ *
+ * @param keepTmpDir
+ * whether or not to remove the scratch dir at the same time.
+ * @throws Exception
*/
- public void close(boolean keepTmpDir) throws TezException, IOException {
+ public void close(boolean keepTmpDir) throws Exception {
if (!isOpen()) {
return;
}
hive git commit: HIVE-10484: Vectorization : RuntimeException Big Table Retained Mapping duplicate column (Matt McCline reviewed by Vikram Dixit)
Repository: hive
Updated Branches:
refs/heads/master 253104075 - bd8d59e40
HIVE-10484: Vectorization : RuntimeException Big Table Retained Mapping
duplicate column (Matt McCline reviewed by Vikram Dixit)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bd8d59e4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bd8d59e4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bd8d59e4
Branch: refs/heads/master
Commit: bd8d59e40ae87fc23c030fd1dda19a4ddfe3cb5b
Parents: 2531040
Author: vikram vik...@hortonworks.com
Authored: Wed May 6 17:27:52 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Wed May 6 17:27:52 2015 -0700
--
.../ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/bd8d59e4/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
index b215f70..a9082eb 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
@@ -321,7 +321,10 @@ public abstract class VectorMapJoinCommonOperator extends
MapJoinOperator implem
projectionMapping.add(nextOutputColumn, batchColumnIndex, typeName);
// Collect columns we copy from the big table batch to the overflow
batch.
- bigTableRetainedMapping.add(batchColumnIndex, batchColumnIndex,
typeName);
+ if (!bigTableRetainedMapping.containsOutputColumn(batchColumnIndex)) {
+// Tolerate repeated use of a big table column.
+bigTableRetainedMapping.add(batchColumnIndex, batchColumnIndex,
typeName);
+ }
nextOutputColumn++;
}
hive git commit: HIVE-10484: Vectorization : RuntimeException Big Table Retained Mapping duplicate column (Matt McCline via Vikram Dixit)
Repository: hive
Updated Branches:
refs/heads/branch-1.2 5c644a22d - 03e22a9d4
HIVE-10484: Vectorization : RuntimeException Big Table Retained Mapping
duplicate column (Matt McCline via Vikram Dixit)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/03e22a9d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/03e22a9d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/03e22a9d
Branch: refs/heads/branch-1.2
Commit: 03e22a9d46c8a75b5a97f050644a8da6e263f2a4
Parents: 5c644a2
Author: vikram vik...@hortonworks.com
Authored: Wed May 6 17:35:42 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Wed May 6 17:35:42 2015 -0700
--
.../ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/03e22a9d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
index b215f70..a9082eb 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
@@ -321,7 +321,10 @@ public abstract class VectorMapJoinCommonOperator extends
MapJoinOperator implem
projectionMapping.add(nextOutputColumn, batchColumnIndex, typeName);
// Collect columns we copy from the big table batch to the overflow
batch.
- bigTableRetainedMapping.add(batchColumnIndex, batchColumnIndex,
typeName);
+ if (!bigTableRetainedMapping.containsOutputColumn(batchColumnIndex)) {
+// Tolerate repeated use of a big table column.
+bigTableRetainedMapping.add(batchColumnIndex, batchColumnIndex,
typeName);
+ }
nextOutputColumn++;
}
hive git commit: HIVE-10609: Vectorization : Q64 fails with ClassCastException (Matt McCline via Vikram Dixit K)
Repository: hive
Updated Branches:
refs/heads/branch-1.2 fe4f7a51c - ef9686bf8
HIVE-10609: Vectorization : Q64 fails with ClassCastException (Matt McCline via
Vikram Dixit K)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ef9686bf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ef9686bf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ef9686bf
Branch: refs/heads/branch-1.2
Commit: ef9686bf8dac78b3071f4a1713bf5cb46366d7b0
Parents: fe4f7a5
Author: vikram vik...@hortonworks.com
Authored: Mon May 11 15:55:21 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Mon May 11 15:55:21 2015 -0700
--
.../hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java| 8 ++--
.../ql/exec/vector/VectorMapJoinOuterFilteredOperator.java | 6 --
2 files changed, 10 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/ef9686bf/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
index 15c747e..e9bd44a 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
@@ -101,15 +101,19 @@ public class VectorMapJoinOperator extends
VectorMapJoinBaseOperator {
@Override
public CollectionFuture? initializeOp(Configuration hconf) throws
HiveException {
+// Use a final variable to properly parameterize the
processVectorInspector closure.
+// Using a member variable in the closure will not do the right thing...
+final int parameterizePosBigTable = conf.getPosBigTable();
+
// Code borrowed from VectorReduceSinkOperator.initializeOp
VectorExpressionWriterFactory.processVectorInspector(
-(StructObjectInspector) inputObjInspectors[0],
+(StructObjectInspector) inputObjInspectors[parameterizePosBigTable],
new VectorExpressionWriterFactory.SingleOIDClosure() {
@Override
public void assign(VectorExpressionWriter[] writers,
ObjectInspector objectInspector) {
rowWriters = writers;
-inputObjInspectors[0] = objectInspector;
+inputObjInspectors[parameterizePosBigTable] = objectInspector;
}
});
singleRow = new Object[rowWriters.length];
http://git-wip-us.apache.org/repos/asf/hive/blob/ef9686bf/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java
index 5aecfcc..a96816f 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java
@@ -62,10 +62,12 @@ public class VectorMapJoinOuterFilteredOperator extends
VectorMapJoinBaseOperato
@Override
public CollectionFuture? initializeOp(Configuration hconf) throws
HiveException {
+final int posBigTable = conf.getPosBigTable();
+
// We need a input object inspector that is for the row we will extract
out of the
// vectorized row batch, not for example, an original inspector for an ORC
table, etc.
-inputObjInspectors[0] =
-
VectorizedBatchUtil.convertToStandardStructObjectInspector((StructObjectInspector)
inputObjInspectors[0]);
+inputObjInspectors[posBigTable] =
+
VectorizedBatchUtil.convertToStandardStructObjectInspector((StructObjectInspector)
inputObjInspectors[posBigTable]);
// Call super VectorMapJoinOuterFilteredOperator, which calls super
MapJoinOperator with
// new input inspector.
[2/2] hive git commit: HIVE-10542: Full outer joins in tez produce incorrect results in certain cases (Vikram Dixit K, reviewed by Gunther Hagleitner)
HIVE-10542: Full outer joins in tez produce incorrect results in certain cases
(Vikram Dixit K, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/26ec033c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/26ec033c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/26ec033c
Branch: refs/heads/master
Commit: 26ec033c89a61fa0bf95b9b66da0842b22ec4c9b
Parents: 4b44408
Author: vikram vik...@hortonworks.com
Authored: Wed May 6 22:18:28 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Wed May 6 22:18:28 2015 -0700
--
.../test/resources/testconfiguration.properties | 2 +-
.../hive/ql/exec/CommonMergeJoinOperator.java | 54 +-
ql/src/test/queries/clientpositive/mergejoin.q | 17 +
.../test/results/clientpositive/mergejoin.q.out | 844 ---
.../clientpositive/tez/auto_join29.q.out| 500 +++
.../results/clientpositive/tez/mergejoin.q.out | 844 ---
6 files changed, 2005 insertions(+), 256 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/26ec033c/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 134fded..3eff7d0 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -124,6 +124,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
mapreduce2.q,\
merge1.q,\
merge2.q,\
+ mergejoin.q,\
metadataonly1.q,\
metadata_only_queries.q,\
optimize_nullscan.q,\
@@ -160,7 +161,6 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
transform1.q,\
transform2.q,\
transform_ppr1.q,\
- mergejoin.q,\
transform_ppr2.q,\
union2.q,\
union3.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/26ec033c/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
index cb0a5e7..d1d5e2b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
@@ -36,6 +39,7 @@ import org.apache.hadoop.hive.ql.exec.tez.TezContext;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -83,6 +87,7 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
transient ListOperator? extends OperatorDesc originalParents =
new ArrayListOperator? extends OperatorDesc();
+ transient SetInteger fetchInputAtClose;
public CommonMergeJoinOperator() {
super();
@@ -93,6 +98,7 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
public CollectionFuture? initializeOp(Configuration hconf) throws
HiveException {
CollectionFuture? result = super.initializeOp(hconf);
firstFetchHappened = false;
+fetchInputAtClose = getFetchInputAtCloseList();
int maxAlias = 0;
for (byte pos = 0; pos order.length; pos++) {
@@ -145,6 +151,25 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
return result;
}
+ /*
+ * In case of outer joins, we need to push records through even if one of
the sides is done
+ * sending records. For e.g. In the case of full outer join, the right side
needs to send in data
+ * for the join even after the left side has completed sending all the
records on its side. This
+ * can be done once at initialize time and at close, these tags will still
forward records until
+ * they have no more to send. Also, subsequent joins need to fetch their
data as well since
+ * any join following the outer join could produce results with one of the
outer sides depending on
+ * the join condition. We could
[2/2] hive git commit: HIVE-10542: Full outer joins in tez produce incorrect results in certain cases (Vikram Dixit K, reviewed by Gunther Hagleitner)
HIVE-10542: Full outer joins in tez produce incorrect results in certain cases
(Vikram Dixit K, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7e0e5cf0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7e0e5cf0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7e0e5cf0
Branch: refs/heads/branch-1.2
Commit: 7e0e5cf0ee5b384398e654bbe8f43f9db5c8e2c0
Parents: 0e380c7
Author: vikram vik...@hortonworks.com
Authored: Wed May 6 22:19:46 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Wed May 6 22:19:46 2015 -0700
--
.../test/resources/testconfiguration.properties | 2 +-
.../hive/ql/exec/CommonMergeJoinOperator.java | 54 +-
ql/src/test/queries/clientpositive/mergejoin.q | 17 +
.../test/results/clientpositive/mergejoin.q.out | 844 ---
.../clientpositive/tez/auto_join29.q.out| 500 +++
.../results/clientpositive/tez/mergejoin.q.out | 844 ---
6 files changed, 2005 insertions(+), 256 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/7e0e5cf0/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 134fded..3eff7d0 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -124,6 +124,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
mapreduce2.q,\
merge1.q,\
merge2.q,\
+ mergejoin.q,\
metadataonly1.q,\
metadata_only_queries.q,\
optimize_nullscan.q,\
@@ -160,7 +161,6 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
transform1.q,\
transform2.q,\
transform_ppr1.q,\
- mergejoin.q,\
transform_ppr2.q,\
union2.q,\
union3.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/7e0e5cf0/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
index cb0a5e7..d1d5e2b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
@@ -36,6 +39,7 @@ import org.apache.hadoop.hive.ql.exec.tez.TezContext;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -83,6 +87,7 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
transient ListOperator? extends OperatorDesc originalParents =
new ArrayListOperator? extends OperatorDesc();
+ transient SetInteger fetchInputAtClose;
public CommonMergeJoinOperator() {
super();
@@ -93,6 +98,7 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
public CollectionFuture? initializeOp(Configuration hconf) throws
HiveException {
CollectionFuture? result = super.initializeOp(hconf);
firstFetchHappened = false;
+fetchInputAtClose = getFetchInputAtCloseList();
int maxAlias = 0;
for (byte pos = 0; pos order.length; pos++) {
@@ -145,6 +151,25 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
return result;
}
+ /*
+ * In case of outer joins, we need to push records through even if one of
the sides is done
+ * sending records. For e.g. In the case of full outer join, the right side
needs to send in data
+ * for the join even after the left side has completed sending all the
records on its side. This
+ * can be done once at initialize time and at close, these tags will still
forward records until
+ * they have no more to send. Also, subsequent joins need to fetch their
data as well since
+ * any join following the outer join could produce results with one of the
outer sides depending on
+ * the join condition. We
[1/2] hive git commit: HIVE-10542: Full outer joins in tez produce incorrect results in certain cases (Vikram Dixit K, reviewed by Gunther Hagleitner)
Repository: hive Updated Branches: refs/heads/branch-1.2 0e380c71c - 7e0e5cf0e http://git-wip-us.apache.org/repos/asf/hive/blob/7e0e5cf0/ql/src/test/results/clientpositive/tez/mergejoin.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/mergejoin.q.out b/ql/src/test/results/clientpositive/tez/mergejoin.q.out index 48cd2a1..97df12a 100644 --- a/ql/src/test/results/clientpositive/tez/mergejoin.q.out +++ b/ql/src/test/results/clientpositive/tez/mergejoin.q.out @@ -1,7 +1,11 @@ -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain select * from src a join src1 b on a.key = b.key PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain select * from src a join src1 b on a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -347,74 +351,12 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 -2 val_2 2008-04-08 2 val_2 2008-04-08 -4 val_4 2008-04-08 4 val_4 2008-04-08 -8 val_8 2008-04-08 8 val_8 2008-04-08 -11 val_11 2008-04-08 11 val_11 2008-04-08 -15 val_15 2008-04-08 15 val_15 2008-04-08 -15 val_15 2008-04-08 15 val_15 2008-04-08 -15 val_15 2008-04-08 15 val_15 2008-04-08 -15 val_15 2008-04-08 15 val_15 2008-04-08 -17 val_17 2008-04-08 17 val_17 2008-04-08 -19 val_19 2008-04-08 19 val_19 2008-04-08 -20 val_20 2008-04-08 20 val_20 2008-04-08 -24 val_24 2008-04-08 24 val_24 2008-04-08 -24 val_24 2008-04-08 24 val_24 2008-04-08 -24 val_24 2008-04-08 24 val_24 2008-04-08 -24 val_24 2008-04-08 24 val_24 2008-04-08 -26 val_26 2008-04-08 26 val_26 2008-04-08 -26 val_26 2008-04-08 26 val_26 2008-04-08 -26 val_26 2008-04-08 26 val_26 2008-04-08 -26 val_26 2008-04-08 26 val_26 2008-04-08 -28 val_28 2008-04-08 28 val_28 2008-04-08 -33 val_33 2008-04-08 33 val_33 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -37 val_37 2008-04-08 37 val_37 2008-04-08 -37 val_37 2008-04-08 37 val_37 2008-04-08 -37 val_37 2008-04-08 37 val_37 2008-04-08 -37 val_37 2008-04-08 37 val_37 2008-04-08 -42 val_42 2008-04-08 42 val_42 2008-04-08 -42 val_42 2008-04-08 42 val_42 2008-04-08 -42 val_42 2008-04-08 42 val_42 2008-04-08 -42 val_42 2008-04-08 42 val_42 2008-04-08 -44 val_44 2008-04-08 44 val_44 2008-04-08 -51 val_51 2008-04-08 51 val_51 2008-04-08 -51 val_51 2008-04-08 51 val_51 2008-04-08 -51 val_51 2008-04-08 51 val_51 2008-04-08 -51 val_51 2008-04-08 51 val_51 2008-04-08 -53 val_53 2008-04-08 53 val_53 2008-04-08 -57 val_57 2008-04-08 57 val_57 2008-04-08 -64 val_64 2008-04-08 64 val_64 2008-04-08 -66 val_66 2008-04-08 66 val_66 2008-04-08 -77 val_77 2008-04-08 77 val_77 2008-04-08 -80 val_80 2008-04-08 80 val_80 2008-04-08 -82 val_82 2008-04-08 82 val_82 2008-04-08 -84 val_84 2008-04-08 84 val_84 2008-04-08 -84 val_84 2008-04-08 84 val_84 2008-04-08 -84 val_84 2008-04-08 84 val_84 2008-04-08 -84 val_84 2008-04-08 84 val_84 2008-04-08 -86 val_86 2008-04-08 86 val_86 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -97 val_97 2008-04-08 97 val_97 2008-04-08 -97 val_97 2008-04-08 97 val_97 2008-04-08 -97 val_97 2008-04-08 97 val_97 2008-04-08 -97 val_97 2008-04-08 97 val_97 2008-04-08 103val_103 2008-04-08 103 val_103 2008-04-08 103val_103 2008-04-08 103 val_103 2008-04-08 103val_103 2008-04-08 103
[1/2] hive git commit: HIVE-10542: Full outer joins in tez produce incorrect results in certain cases (Vikram Dixit K, reviewed by Gunther Hagleitner)
Repository: hive Updated Branches: refs/heads/master 4b444082f - 26ec033c8 http://git-wip-us.apache.org/repos/asf/hive/blob/26ec033c/ql/src/test/results/clientpositive/tez/mergejoin.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/mergejoin.q.out b/ql/src/test/results/clientpositive/tez/mergejoin.q.out index 48cd2a1..97df12a 100644 --- a/ql/src/test/results/clientpositive/tez/mergejoin.q.out +++ b/ql/src/test/results/clientpositive/tez/mergejoin.q.out @@ -1,7 +1,11 @@ -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain select * from src a join src1 b on a.key = b.key PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain select * from src a join src1 b on a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -347,74 +351,12 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 0 val_0 2008-04-08 -2 val_2 2008-04-08 2 val_2 2008-04-08 -4 val_4 2008-04-08 4 val_4 2008-04-08 -8 val_8 2008-04-08 8 val_8 2008-04-08 -11 val_11 2008-04-08 11 val_11 2008-04-08 -15 val_15 2008-04-08 15 val_15 2008-04-08 -15 val_15 2008-04-08 15 val_15 2008-04-08 -15 val_15 2008-04-08 15 val_15 2008-04-08 -15 val_15 2008-04-08 15 val_15 2008-04-08 -17 val_17 2008-04-08 17 val_17 2008-04-08 -19 val_19 2008-04-08 19 val_19 2008-04-08 -20 val_20 2008-04-08 20 val_20 2008-04-08 -24 val_24 2008-04-08 24 val_24 2008-04-08 -24 val_24 2008-04-08 24 val_24 2008-04-08 -24 val_24 2008-04-08 24 val_24 2008-04-08 -24 val_24 2008-04-08 24 val_24 2008-04-08 -26 val_26 2008-04-08 26 val_26 2008-04-08 -26 val_26 2008-04-08 26 val_26 2008-04-08 -26 val_26 2008-04-08 26 val_26 2008-04-08 -26 val_26 2008-04-08 26 val_26 2008-04-08 -28 val_28 2008-04-08 28 val_28 2008-04-08 -33 val_33 2008-04-08 33 val_33 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -35 val_35 2008-04-08 35 val_35 2008-04-08 -37 val_37 2008-04-08 37 val_37 2008-04-08 -37 val_37 2008-04-08 37 val_37 2008-04-08 -37 val_37 2008-04-08 37 val_37 2008-04-08 -37 val_37 2008-04-08 37 val_37 2008-04-08 -42 val_42 2008-04-08 42 val_42 2008-04-08 -42 val_42 2008-04-08 42 val_42 2008-04-08 -42 val_42 2008-04-08 42 val_42 2008-04-08 -42 val_42 2008-04-08 42 val_42 2008-04-08 -44 val_44 2008-04-08 44 val_44 2008-04-08 -51 val_51 2008-04-08 51 val_51 2008-04-08 -51 val_51 2008-04-08 51 val_51 2008-04-08 -51 val_51 2008-04-08 51 val_51 2008-04-08 -51 val_51 2008-04-08 51 val_51 2008-04-08 -53 val_53 2008-04-08 53 val_53 2008-04-08 -57 val_57 2008-04-08 57 val_57 2008-04-08 -64 val_64 2008-04-08 64 val_64 2008-04-08 -66 val_66 2008-04-08 66 val_66 2008-04-08 -77 val_77 2008-04-08 77 val_77 2008-04-08 -80 val_80 2008-04-08 80 val_80 2008-04-08 -82 val_82 2008-04-08 82 val_82 2008-04-08 -84 val_84 2008-04-08 84 val_84 2008-04-08 -84 val_84 2008-04-08 84 val_84 2008-04-08 -84 val_84 2008-04-08 84 val_84 2008-04-08 -84 val_84 2008-04-08 84 val_84 2008-04-08 -86 val_86 2008-04-08 86 val_86 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -97 val_97 2008-04-08 97 val_97 2008-04-08 -97 val_97 2008-04-08 97 val_97 2008-04-08 -97 val_97 2008-04-08 97 val_97 2008-04-08 -97 val_97 2008-04-08 97 val_97 2008-04-08 103val_103 2008-04-08 103 val_103 2008-04-08 103val_103 2008-04-08 103 val_103 2008-04-08 103val_103 2008-04-08 103
[4/4] hive git commit: HIVE-10542: Full outer joins in tez produce incorrect results in certain cases (Vikram Dixit K, reviewed by Gunther Hagleitner)
HIVE-10542: Full outer joins in tez produce incorrect results in certain cases
(Vikram Dixit K, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db56e8e3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db56e8e3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db56e8e3
Branch: refs/heads/branch-1.0
Commit: db56e8e3d0285068c789fdb3402d99aee41f553f
Parents: 5a9fddb
Author: vikram vik...@hortonworks.com
Authored: Wed May 13 12:17:18 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Wed May 13 12:17:18 2015 -0700
--
.../test/resources/testconfiguration.properties |1 +
.../hive/ql/exec/CommonMergeJoinOperator.java | 55 +-
ql/src/test/queries/clientpositive/mergejoin.q | 124 +
.../test/results/clientpositive/mergejoin.q.out | 3235 ++
.../clientpositive/tez/auto_join29.q.out| 500 +++
.../results/clientpositive/tez/mergejoin.q.out | 3184 +
6 files changed, 7098 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/db56e8e3/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 7898d81..8f96826 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -107,6 +107,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
mapreduce2.q,\
merge1.q,\
merge2.q,\
+ mergejoin.q,\
metadataonly1.q,\
metadata_only_queries.q,\
optimize_nullscan.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/db56e8e3/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
index 1da8933..fb97774 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,6 +35,8 @@ import org.apache.hadoop.hive.ql.exec.tez.RecordSource;
import org.apache.hadoop.hive.ql.exec.tez.TezContext;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
+import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
+import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -78,6 +82,7 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
transient RecordSource[] sources;
transient ListOperator? extends OperatorDesc originalParents =
new ArrayListOperator? extends OperatorDesc();
+ transient SetInteger fetchInputAtClose;
public CommonMergeJoinOperator() {
super();
@@ -88,6 +93,8 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
public void initializeOp(Configuration hconf) throws HiveException {
super.initializeOp(hconf);
firstFetchHappened = false;
+fetchInputAtClose = getFetchInputAtCloseList();
+
initializeChildren(hconf);
int maxAlias = 0;
for (byte pos = 0; pos order.length; pos++) {
@@ -134,6 +141,25 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperatorCommonMerge
sources = ((TezContext) MapredContext.get()).getRecordSources();
}
+ /*
+ * In case of outer joins, we need to push records through even if one of
the sides is done
+ * sending records. For e.g. In the case of full outer join, the right side
needs to send in data
+ * for the join even after the left side has completed sending all the
records on its side. This
+ * can be done once at initialize time and at close, these tags will still
forward records until
+ * they have no more to send. Also, subsequent joins need to fetch their
data as well since
+ * any join following the outer join could produce results with one of the
outer sides depending on
+ * the join condition. We could optimize for the case of inner joins in the
future here.
+ */
+ private SetInteger getFetchInputAtCloseList
[3/4] hive git commit: HIVE-10542: Full outer joins in tez produce incorrect results in certain cases (Vikram Dixit K, reviewed by Gunther Hagleitner)
http://git-wip-us.apache.org/repos/asf/hive/blob/db56e8e3/ql/src/test/results/clientpositive/mergejoin.q.out
--
diff --git a/ql/src/test/results/clientpositive/mergejoin.q.out
b/ql/src/test/results/clientpositive/mergejoin.q.out
new file mode 100644
index 000..e2a5bfa
--- /dev/null
+++ b/ql/src/test/results/clientpositive/mergejoin.q.out
@@ -0,0 +1,3235 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
+select * from src a join src1 b on a.key = b.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
+select * from src a join src1 b on a.key = b.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+Map Reduce
+ Map Operator Tree:
+ TableScan
+alias: b
+filterExpr: key is not null (type: boolean)
+Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE
Column stats: NONE
+Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+key expressions: key (type: string)
+sort order: +
+Map-reduce partition columns: key (type: string)
+Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE
Column stats: NONE
+value expressions: value (type: string)
+ TableScan
+alias: a
+filterExpr: key is not null (type: boolean)
+Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
+Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+key expressions: key (type: string)
+sort order: +
+Map-reduce partition columns: key (type: string)
+Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
+value expressions: value (type: string)
+ Reduce Operator Tree:
+Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+0 {KEY.reducesinkkey0} {VALUE._col0}
+1 {KEY.reducesinkkey0} {VALUE._col0}
+ outputColumnNames: _col0, _col1, _col5, _col6
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+expressions: _col0 (type: string), _col1 (type: string), _col5
(type: string), _col6 (type: string)
+outputColumnNames: _col0, _col1, _col2, _col3
+Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE
Column stats: NONE
+File Output Operator
+ compressed: false
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+Fetch Operator
+ limit: -1
+ Processor Tree:
+ListSink
+
+PREHOOK: query: select * from src a join src1 b on a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+ A masked pattern was here
+POSTHOOK: query: select * from src a join src1 b on a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+ A masked pattern was here
+128val_128 128
+128val_128 128
+128val_128 128
+146val_146 146 val_146
+146val_146 146 val_146
+150val_150 150 val_150
+213val_213 213 val_213
+213val_213 213 val_213
+224val_224 224
+224val_224 224
+238val_238 238 val_238
+238val_238 238 val_238
+255val_255 255 val_255
+255val_255 255 val_255
+273val_273 273 val_273
+273val_273 273 val_273
+273val_273 273 val_273
+278val_278 278 val_278
+278val_278 278 val_278
+311val_311 311 val_311
+311val_311 311 val_311
+311val_311 311 val_311
+369val_369 369
+369val_369 369
+369val_369 369
+401val_401 401 val_401
+401val_401 401 val_401
+401val_401 401 val_401
+401val_401 401 val_401
+401val_401 401 val_401
+406val_406 406 val_406
+406val_406 406 val_406
+406val_406 406 val_406
+406val_406 406 val_406
+66 val_66 66 val_66
+98 val_98 98 val_98
+98 val_98
[2/4] hive git commit: HIVE-10542: Full outer joins in tez produce incorrect results in certain cases (Vikram Dixit K, reviewed by Gunther Hagleitner)
http://git-wip-us.apache.org/repos/asf/hive/blob/db56e8e3/ql/src/test/results/clientpositive/tez/auto_join29.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/auto_join29.q.out b/ql/src/test/results/clientpositive/tez/auto_join29.q.out index 80c70ef..59b08ef 100644 --- a/ql/src/test/results/clientpositive/tez/auto_join29.q.out +++ b/ql/src/test/results/clientpositive/tez/auto_join29.q.out @@ -2669,6 +2669,506 @@ POSTHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AN POSTHOOK: type: QUERY POSTHOOK: Input: default@src A masked pattern was here +NULL NULLNULLNULL0 val_0 +NULL NULLNULLNULL0 val_0 +NULL NULLNULLNULL0 val_0 +NULL NULLNULLNULL10 val_10 +NULL NULLNULLNULL100 val_100 +NULL NULLNULLNULL100 val_100 +NULL NULLNULLNULL103 val_103 +NULL NULLNULLNULL103 val_103 +NULL NULLNULLNULL104 val_104 +NULL NULLNULLNULL104 val_104 +NULL NULLNULLNULL105 val_105 +NULL NULLNULLNULL11 val_11 +NULL NULLNULLNULL111 val_111 +NULL NULLNULLNULL113 val_113 +NULL NULLNULLNULL113 val_113 +NULL NULLNULLNULL114 val_114 +NULL NULLNULLNULL116 val_116 +NULL NULLNULLNULL118 val_118 +NULL NULLNULLNULL118 val_118 +NULL NULLNULLNULL119 val_119 +NULL NULLNULLNULL119 val_119 +NULL NULLNULLNULL119 val_119 +NULL NULLNULLNULL12 val_12 +NULL NULLNULLNULL12 val_12 +NULL NULLNULLNULL120 val_120 +NULL NULLNULLNULL120 val_120 +NULL NULLNULLNULL125 val_125 +NULL NULLNULLNULL125 val_125 +NULL NULLNULLNULL126 val_126 +NULL NULLNULLNULL128 val_128 +NULL NULLNULLNULL128 val_128 +NULL NULLNULLNULL128 val_128 +NULL NULLNULLNULL129 val_129 +NULL NULLNULLNULL129 val_129 +NULL NULLNULLNULL131 val_131 +NULL NULLNULLNULL133 val_133 +NULL NULLNULLNULL134 val_134 +NULL NULLNULLNULL134 val_134 +NULL NULLNULLNULL136 val_136 +NULL NULLNULLNULL137 val_137 +NULL NULLNULLNULL137 val_137 +NULL NULLNULLNULL138 val_138 +NULL NULLNULLNULL138 val_138 +NULL NULLNULLNULL138 val_138 +NULL NULLNULLNULL138 val_138 +NULL NULLNULLNULL143 val_143 +NULL NULLNULLNULL145 val_145 +NULL NULLNULLNULL146 val_146 +NULL NULLNULLNULL146 val_146 +NULL NULLNULLNULL149 val_149 +NULL NULLNULLNULL149 val_149 +NULL NULLNULLNULL15 val_15 +NULL NULLNULLNULL15 val_15 +NULL NULLNULLNULL150 val_150 +NULL NULLNULLNULL152 val_152 +NULL NULLNULLNULL152 val_152 +NULL NULLNULLNULL153 val_153 +NULL NULLNULLNULL155 val_155 +NULL NULLNULLNULL156 val_156 +NULL NULLNULLNULL157 val_157 +NULL NULLNULLNULL158 val_158 +NULL NULLNULLNULL160 val_160 +NULL NULLNULLNULL162 val_162 +NULL NULLNULLNULL163 val_163 +NULL NULLNULLNULL164 val_164 +NULL NULLNULLNULL164 val_164 +NULL NULLNULLNULL165 val_165 +NULL NULLNULLNULL165 val_165 +NULL NULLNULLNULL166 val_166 +NULL NULLNULLNULL167 val_167 +NULL NULLNULLNULL167 val_167 +NULL NULLNULLNULL167 val_167 +NULL NULLNULLNULL168 val_168 +NULL NULLNULLNULL169 val_169 +NULL NULLNULLNULL169 val_169 +NULL NULLNULLNULL169 val_169 +NULL NULLNULLNULL169 val_169 +NULL NULLNULLNULL17 val_17 +NULL NULLNULLNULL170 val_170 +NULL NULLNULLNULL172 val_172 +NULL NULLNULLNULL172 val_172 +NULL NULLNULLNULL174 val_174 +NULL NULLNULLNULL174 val_174 +NULL NULLNULLNULL175 val_175 +NULL NULLNULLNULL175 val_175 +NULL NULLNULLNULL176 val_176 +NULL NULLNULLNULL176 val_176 +NULL NULLNULLNULL177 val_177 +NULL NULLNULLNULL178 val_178 +NULL NULLNULLNULL179 val_179 +NULL NULLNULL
svn commit: r9016 - in /release/hive: ./ hive-1.2.0/
Author: vikram Date: Fri May 15 22:36:58 2015 New Revision: 9016 Log: Add hive-1.2.0 to svn dist Added: release/hive/hive-1.2.0/ release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz (with props) release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz.asc (with props) release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz.md5 release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz (with props) release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz.asc (with props) release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz.md5 Modified: release/hive/stable Added: release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz == Binary file - no diff available. Propchange: release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz -- svn:mime-type = application/x-gzip Added: release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz.asc == Binary file - no diff available. Propchange: release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz.md5 == --- release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz.md5 (added) +++ release/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz.md5 Fri May 15 22:36:58 2015 @@ -0,0 +1 @@ +17871eea4d087695ac5d0d03386e4ec2 apache-hive-1.2.0-bin.tar.gz Added: release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz == Binary file - no diff available. Propchange: release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz -- svn:mime-type = application/x-gzip Added: release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz.asc == Binary file - no diff available. Propchange: release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz.md5 == --- release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz.md5 (added) +++ release/hive/hive-1.2.0/apache-hive-1.2.0-src.tar.gz.md5 Fri May 15 22:36:58 2015 @@ -0,0 +1 @@ +55f3bf70ec40041ecad8dcdc0435d6b6 apache-hive-1.2.0-src.tar.gz Modified: release/hive/stable == --- release/hive/stable (original) +++ release/hive/stable Fri May 15 22:36:58 2015 @@ -1 +1 @@ -link hive-1.1.0 \ No newline at end of file +link hive-1.2.0 \ No newline at end of file
hive git commit: HIVE-10719: Hive metastore failure when alter table rename is attempted. (Vikram Dixit K, reviewed by Thejas Nair, Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 129c496bd - 325a592f5
HIVE-10719: Hive metastore failure when alter table rename is attempted.
(Vikram Dixit K, reviewed by Thejas Nair, Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/325a592f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/325a592f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/325a592f
Branch: refs/heads/master
Commit: 325a592f5f670bef6d6921161ef2d511d21c07d8
Parents: 129c496
Author: vikram vik...@hortonworks.com
Authored: Mon May 18 12:46:49 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Mon May 18 13:52:46 2015 -0700
--
.../hadoop/hive/metastore/HiveAlterHandler.java | 4 +--
.../clientpositive/alter_change_db_location.q | 5 +++
.../alter_change_db_location.q.out | 35
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/325a592f/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index a3f2359..5391171 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -175,8 +175,8 @@ public class HiveAlterHandler implements AlterHandler {
// that means user is asking metastore to move data to new location
// corresponding to the new name
// get new location
-Path databasePath = constructRenamedPath(
-wh.getDefaultDatabasePath(newt.getDbName()), srcPath);
+Database db = msdb.getDatabase(newt.getDbName());
+Path databasePath = constructRenamedPath(wh.getDatabasePath(db),
srcPath);
destPath = new Path(databasePath, newt.getTableName());
destFs = wh.getFs(destPath);
http://git-wip-us.apache.org/repos/asf/hive/blob/325a592f/ql/src/test/queries/clientpositive/alter_change_db_location.q
--
diff --git a/ql/src/test/queries/clientpositive/alter_change_db_location.q
b/ql/src/test/queries/clientpositive/alter_change_db_location.q
new file mode 100644
index 000..a74ba2a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alter_change_db_location.q
@@ -0,0 +1,5 @@
+create database newDB location /tmp/;
+describe database extended newDB;
+use newDB;
+create table tab (name string);
+alter table tab rename to newName;
http://git-wip-us.apache.org/repos/asf/hive/blob/325a592f/ql/src/test/results/clientpositive/alter_change_db_location.q.out
--
diff --git a/ql/src/test/results/clientpositive/alter_change_db_location.q.out
b/ql/src/test/results/clientpositive/alter_change_db_location.q.out
new file mode 100644
index 000..64bd680
--- /dev/null
+++ b/ql/src/test/results/clientpositive/alter_change_db_location.q.out
@@ -0,0 +1,35 @@
+ A masked pattern was here
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:newDB
+ A masked pattern was here
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:newDB
+ A masked pattern was here
+PREHOOK: query: describe database extended newDB
+PREHOOK: type: DESCDATABASE
+POSTHOOK: query: describe database extended newDB
+POSTHOOK: type: DESCDATABASE
+newdb location/in/testhive_test_user USER
+PREHOOK: query: use newDB
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:newdb
+POSTHOOK: query: use newDB
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:newdb
+PREHOOK: query: create table tab (name string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:newdb
+PREHOOK: Output: newDB@tab
+POSTHOOK: query: create table tab (name string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:newdb
+POSTHOOK: Output: newDB@tab
+PREHOOK: query: alter table tab rename to newName
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: newdb@tab
+PREHOOK: Output: newdb@tab
+POSTHOOK: query: alter table tab rename to newName
+POSTHOOK: type: ALTERTABLE_RENAME
+POSTHOOK: Input: newdb@tab
+POSTHOOK: Output: newDB@newName
+POSTHOOK: Output: newdb@tab
hive git commit: HIVE-10719: Hive metastore failure when alter table rename is attempted. (Vikram Dixit K, reviewed by Thejas Nair, Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/branch-1.2 8fece5b4a - 6644ae3d2
HIVE-10719: Hive metastore failure when alter table rename is attempted.
(Vikram Dixit K, reviewed by Thejas Nair, Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6644ae3d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6644ae3d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6644ae3d
Branch: refs/heads/branch-1.2
Commit: 6644ae3d2b42c3622197f876a10e536241c82ee4
Parents: 8fece5b
Author: vikram vik...@hortonworks.com
Authored: Mon May 18 13:56:15 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Mon May 18 13:56:15 2015 -0700
--
.../hadoop/hive/metastore/HiveAlterHandler.java | 4 +--
.../clientpositive/alter_change_db_location.q | 5 +++
.../alter_change_db_location.q.out | 35
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/6644ae3d/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 50ec1e8..d0351da 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -174,8 +174,8 @@ public class HiveAlterHandler implements AlterHandler {
// that means user is asking metastore to move data to new location
// corresponding to the new name
// get new location
-Path databasePath = constructRenamedPath(
-wh.getDefaultDatabasePath(newt.getDbName()), srcPath);
+Database db = msdb.getDatabase(newt.getDbName());
+Path databasePath = constructRenamedPath(wh.getDatabasePath(db),
srcPath);
destPath = new Path(databasePath, newt.getTableName());
destFs = wh.getFs(destPath);
http://git-wip-us.apache.org/repos/asf/hive/blob/6644ae3d/ql/src/test/queries/clientpositive/alter_change_db_location.q
--
diff --git a/ql/src/test/queries/clientpositive/alter_change_db_location.q
b/ql/src/test/queries/clientpositive/alter_change_db_location.q
new file mode 100644
index 000..a74ba2a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alter_change_db_location.q
@@ -0,0 +1,5 @@
+create database newDB location /tmp/;
+describe database extended newDB;
+use newDB;
+create table tab (name string);
+alter table tab rename to newName;
http://git-wip-us.apache.org/repos/asf/hive/blob/6644ae3d/ql/src/test/results/clientpositive/alter_change_db_location.q.out
--
diff --git a/ql/src/test/results/clientpositive/alter_change_db_location.q.out
b/ql/src/test/results/clientpositive/alter_change_db_location.q.out
new file mode 100644
index 000..64bd680
--- /dev/null
+++ b/ql/src/test/results/clientpositive/alter_change_db_location.q.out
@@ -0,0 +1,35 @@
+ A masked pattern was here
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:newDB
+ A masked pattern was here
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:newDB
+ A masked pattern was here
+PREHOOK: query: describe database extended newDB
+PREHOOK: type: DESCDATABASE
+POSTHOOK: query: describe database extended newDB
+POSTHOOK: type: DESCDATABASE
+newdb location/in/testhive_test_user USER
+PREHOOK: query: use newDB
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:newdb
+POSTHOOK: query: use newDB
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:newdb
+PREHOOK: query: create table tab (name string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:newdb
+PREHOOK: Output: newDB@tab
+POSTHOOK: query: create table tab (name string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:newdb
+POSTHOOK: Output: newDB@tab
+PREHOOK: query: alter table tab rename to newName
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: newdb@tab
+PREHOOK: Output: newdb@tab
+POSTHOOK: query: alter table tab rename to newName
+POSTHOOK: type: ALTERTABLE_RENAME
+POSTHOOK: Input: newdb@tab
+POSTHOOK: Output: newDB@newName
+POSTHOOK: Output: newdb@tab
svn commit: r1673937 [1/3] - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/test/queries/client
Author: vikram
Date: Wed Apr 15 19:36:13 2015
New Revision: 1673937
URL: http://svn.apache.org/r1673937
Log:
HIVE-10273: Union with partition tables which have no data fails with NPE
(Vikram Dixit, reviewed by Gunther Hagleitner)
Added:
hive/trunk/ql/src/test/results/clientpositive/tez/union_view.q.out
Modified:
hive/trunk/itests/src/test/resources/testconfiguration.properties
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
hive/trunk/ql/src/test/queries/clientpositive/tez_union.q
hive/trunk/ql/src/test/results/clientnegative/join_nonexistent_part.q.out
hive/trunk/ql/src/test/results/clientpositive/annotate_stats_join.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join32.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer3.q.out
hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_sort_6.q.out
hive/trunk/ql/src/test/results/clientpositive/input23.q.out
hive/trunk/ql/src/test/results/clientpositive/input26.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out
hive/trunk/ql/src/test/results/clientpositive/join_view.q.out
hive/trunk/ql/src/test/results/clientpositive/metadataonly1.q.out
hive/trunk/ql/src/test/results/clientpositive/nullgroup5.q.out
hive/trunk/ql/src/test/results/clientpositive/optimize_nullscan.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_boolexpr.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_union_view.q.out
hive/trunk/ql/src/test/results/clientpositive/reduce_deduplicate.q.out
hive/trunk/ql/src/test/results/clientpositive/sample6.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin9.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/auto_join32.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/join_view.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/sample6.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/union_view.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_union.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
hive/trunk/ql/src/test/results/clientpositive/union30.q.out
hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
hive/trunk/ql/src/test/results/clientpositive/union_view.q.out
Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1673937r1=1673936r2=1673937view=diff
==
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Wed Apr
15 19:36:13 2015
@@ -313,6 +313,7 @@ minitez.query.files=bucket_map_join_tez1
tez_schema_evolution.q,\
tez_union.q,\
tez_union2.q,\
+ tez_union_view.q,\
tez_union_decimal.q,\
tez_union_group_by.q,\
tez_smb_main.q,\
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1673937r1=1673936r2=1673937view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
Wed Apr 15 19:36:13 2015
@@ -365,6 +365,17 @@ public class Vectorizer implements Physi
addMapWorkRules(opRules, vnp);
Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
GraphWalker ogw = new DefaultGraphWalker(disp);
+ if ((mapWork.getAliasToWork() == null) ||
(mapWork.getAliasToWork().size() == 0)) {
+return false;
+ } else {
+for (Operator? op : mapWork.getAliasToWork().values()) {
+ if (op == null) {
+LOG.warn
svn commit: r1673937 [2/3] - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/test/queries/client
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out?rev=1673937r1=1673936r2=1673937view=diff == --- hive/trunk/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out Wed Apr 15 19:36:13 2015 @@ -393,6 +393,40 @@ STAGE PLANS: A masked pattern was here Vertices: Map 1 +Map Operator Tree: +TableScan + alias: complex_tbl_2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + GatherStats: false + Filter Operator +isSamplingPred: false +predicate: (ds = '2010-03-29') (type: boolean) +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Select Operator + expressions: aet (type: string), aes (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Transform Operator +command: cat +output info: +input format: org.apache.hadoop.mapred.TextInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +properties: + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6 + columns.types string,string,int,string,bigint,string,string + field.delim 9 + serialization.format 9 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + auto parallelism: false Reducer 2 Needs Tagging: false Reduce Operator Tree: Modified: hive/trunk/ql/src/test/results/clientpositive/spark/sample6.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/sample6.q.out?rev=1673937r1=1673936r2=1673937view=diff == --- hive/trunk/ql/src/test/results/clientpositive/spark/sample6.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/spark/sample6.q.out Wed Apr 15 19:36:13 2015 @@ -3081,6 +3081,26 @@ STAGE PLANS: A masked pattern was here Vertices: Map 1 +Map Operator Tree: +TableScan + alias: s + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + GatherStats: false + Filter Operator +isSamplingPred: true +predicate: (((hash(key) 2147483647) % 2) = 0) (type: boolean) +sampleDesc: BUCKET 1 OUT OF 2 +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator +key expressions: _col0 (type: int), _col1 (type: string) +sort order: ++ +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +tag: -1 +auto parallelism: false Reducer 2 Needs Tagging: false Reduce Operator Tree: Modified: hive/trunk/ql/src/test/results/clientpositive/spark/union_view.q.out URL:
svn commit: r1673937 [3/3] - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/test/queries/client
Modified: hive/trunk/ql/src/test/results/clientpositive/union_view.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_view.q.out?rev=1673937r1=1673936r2=1673937view=diff == --- hive/trunk/ql/src/test/results/clientpositive/union_view.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/union_view.q.out Wed Apr 15 19:36:13 2015 @@ -503,6 +503,54 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan +alias: src_union_2 +filterExpr: ((key = 86) and (ds = '1')) (type: boolean) +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Filter Operator + predicate: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator +expressions: value (type: string) +outputColumnNames: _col1 +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Union + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: 86 (type: int), _col1 (type: string), '1' (type: string) +outputColumnNames: _col0, _col1, _col2 +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan +alias: src_union_3 +filterExpr: ((key = 86) and (ds = '1')) (type: boolean) +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Filter Operator + predicate: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator +expressions: value (type: string) +outputColumnNames: _col1 +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Union + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: 86 (type: int), _col1 (type: string), '1' (type: string) +outputColumnNames: _col0, _col1, _col2 +Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -571,6 +619,30 @@ STAGE PLANS: Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan +alias: src_union_1 +filterExpr: ((key = 86) and (ds = '2')) (type: boolean) +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Filter Operator + predicate: ((key = 86) and (ds = '2')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator +expressions: value (type: string) +outputColumnNames: _col1 +Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: 86 (type: int), _col1 (type: string), '2' (type: string) +outputColumnNames: _col0, _col1, _col2 +
svn commit: r1670779 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/exec/tez/ java/org/apache/hadoop/hive/ql/optimizer/ java/org/apache/hadoop/hive/ql/opt
Author: vikram
Date: Wed Apr 1 21:45:34 2015
New Revision: 1670779
URL: http://svn.apache.org/r1670779
Log:
HIVE-10001: SMB join in reduce side (Vikram Dixit K, reviewed by Gunther
Hagleitner)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
hive/trunk/ql/src/test/queries/clientpositive/tez_join.q
hive/trunk/ql/src/test/queries/clientpositive/tez_smb_1.q
hive/trunk/ql/src/test/queries/clientpositive/tez_smb_main.q
hive/trunk/ql/src/test/results/clientpositive/tez/tez_join.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/tez_smb_main.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java?rev=1670779r1=1670778r2=1670779view=diff
==
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
Wed Apr 1 21:45:34 2015
@@ -501,12 +501,13 @@ public class CommonMergeJoinOperator ext
if (parent == null) {
throw new HiveException(No valid parents.);
}
-MapInteger, DummyStoreOperator dummyOps = parent.getTagToOperatorTree();
+MapInteger, DummyStoreOperator dummyOps =
+((TezContext) (MapredContext.get())).getDummyOpsMap();
for (EntryInteger, DummyStoreOperator connectOp : dummyOps.entrySet()) {
if (connectOp.getValue().getChildOperators() == null
- || connectOp.getValue().getChildOperators().isEmpty()) {
- parentOperators.add(connectOp.getKey(), connectOp.getValue());
- connectOp.getValue().getChildOperators().add(this);
+ || connectOp.getValue().getChildOperators().isEmpty()) {
+parentOperators.add(connectOp.getKey(), connectOp.getValue());
+connectOp.getValue().getChildOperators().add(this);
}
}
super.initializeLocalWork(hconf);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1670779r1=1670778r2=1670779view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Wed
Apr 1 21:45:34 2015
@@ -637,11 +637,6 @@ public class MapOperator extends Operato
return null;
}
- @Override
- public MapInteger, DummyStoreOperator getTagToOperatorTree() {
-return MapRecordProcessor.getConnectOps();
- }
-
public void initializeContexts() {
Path fpath = getExecContext().getCurrentInputPath();
String nominalPath = getNominalPath(fpath);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1670779r1=1670778r2=1670779view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Wed Apr
svn commit: r9065 - in /release/hive: hive-1.0.1/ hive-1.1.1/
Author: vikram Date: Thu May 21 01:21:09 2015 New Revision: 9065 Log: Add hive-1.1.1 and hive-1.0.1 to svn dist (Chao Sun via Vikram Dixit) Added: release/hive/hive-1.0.1/ release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz (with props) release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz.asc (with props) release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz.md5 release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz (with props) release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz.asc (with props) release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz.md5 release/hive/hive-1.1.1/ release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz (with props) release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz.asc (with props) release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz.md5 release/hive/hive-1.1.1/apache-hive-1.1.1-src.tar.gz (with props) release/hive/hive-1.1.1/apache-hive-1.1.1-src.tar.gz.asc (with props) release/hive/hive-1.1.1/apache-hive-1.1.1-src.tar.gz.md5 Added: release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz == Binary file - no diff available. Propchange: release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz -- svn:mime-type = application/x-gzip Added: release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz.asc == Binary file - no diff available. Propchange: release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz.md5 == --- release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz.md5 (added) +++ release/hive/hive-1.0.1/apache-hive-1.0.1-bin.tar.gz.md5 Thu May 21 01:21:09 2015 @@ -0,0 +1 @@ +MD5 (apache-hive-1.0.1-bin.tar.gz) = d42955c70c0184d6ec60c83ff94ba33c Added: release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz == Binary file - no diff available. Propchange: release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz -- svn:mime-type = application/x-gzip Added: release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz.asc == Binary file - no diff available. Propchange: release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz.md5 == --- release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz.md5 (added) +++ release/hive/hive-1.0.1/apache-hive-1.0.1-src.tar.gz.md5 Thu May 21 01:21:09 2015 @@ -0,0 +1 @@ +MD5 (apache-hive-1.0.1-src.tar.gz) = f73c3f24a0f5f5189ecd8f43382d7158 Added: release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz == Binary file - no diff available. Propchange: release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz -- svn:mime-type = application/x-gzip Added: release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz.asc == Binary file - no diff available. Propchange: release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz.md5 == --- release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz.md5 (added) +++ release/hive/hive-1.1.1/apache-hive-1.1.1-bin.tar.gz.md5 Thu May 21 01:21:09 2015 @@ -0,0 +1 @@ +MD5 (apache-hive-1.1.1-bin.tar.gz) = cdfe203c110b8fc3120af632e768f011 Added: release/hive/hive-1.1.1/apache-hive-1.1.1-src.tar.gz == Binary file - no diff available. Propchange: release/hive/hive-1.1.1/apache-hive-1.1.1-src.tar.gz -- svn:mime-type = application/x-gzip Added: release/hive/hive-1.1.1/apache-hive-1.1.1-src.tar.gz.asc == Binary file - no diff available. Propchange: release/hive/hive-1.1.1/apache-hive-1.1.1-src.tar.gz.asc
hive git commit: HIVE-10736: HiveServer2 shutdown of cached tez app-masters is not clean (Vikram Dixit K, reviewed by Gunther Hagleitner)
Repository: hive
Updated Branches:
refs/heads/branch-1.2 a4d0298f1 - 52635cbe1
HIVE-10736: HiveServer2 shutdown of cached tez app-masters is not clean (Vikram
Dixit K, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52635cbe
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52635cbe
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52635cbe
Branch: refs/heads/branch-1.2
Commit: 52635cbe1363e6f975b616b946871430e6fca781
Parents: a4d0298
Author: vikram vik...@hortonworks.com
Authored: Sat Jun 6 10:57:51 2015 -0700
Committer: vikram vik...@hortonworks.com
Committed: Sat Jun 6 10:57:51 2015 -0700
--
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 2 +-
.../hive/ql/exec/tez/TezSessionPoolManager.java | 21 ++--
.../hive/ql/exec/tez/TezSessionState.java | 10 --
3 files changed, 20 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/52635cbe/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index 78caba8..6c57063 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@ -127,7 +127,7 @@ public class TezJobMonitor {
TezJobMonitor.killRunningJobs();
}
try {
- for (TezSessionState s: TezSessionState.getOpenSessions()) {
+ for (TezSessionState s :
TezSessionPoolManager.getInstance().getOpenSessions()) {
System.err.println(Shutting down tez session.);
TezSessionPoolManager.getInstance().close(s, false);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52635cbe/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
index 1798201..dfa539f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
@@ -20,13 +20,16 @@ package org.apache.hadoop.hive.ql.exec.tez;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -49,6 +52,9 @@ public class TezSessionPoolManager {
private static TezSessionPoolManager sessionPool = null;
+ private static ListTezSessionState openSessions = Collections
+ .synchronizedList(new LinkedListTezSessionState());
+
public static TezSessionPoolManager getInstance()
throws Exception {
if (sessionPool == null) {
@@ -68,6 +74,7 @@ public class TezSessionPoolManager {
TezSessionState sessionState = defaultQueuePool.take();
newConf.set(tez.queue.name, sessionState.getQueueName());
sessionState.open(newConf);
+ openSessions.add(sessionState);
defaultQueuePool.put(sessionState);
}
}
@@ -148,6 +155,7 @@ public class TezSessionPoolManager {
String what = Created;
if (doOpen) {
retTezSessionState.open(conf);
+ openSessions.add(retTezSessionState);
what = Started;
}
@@ -175,6 +183,7 @@ public class TezSessionPoolManager {
LOG.info(Closing tez session default? + tezSessionState.isDefault());
if (!tezSessionState.isDefault()) {
tezSessionState.close(keepTmpDir);
+ openSessions.remove(tezSessionState);
}
}
@@ -184,9 +193,12 @@ public class TezSessionPoolManager {
}
// we can just stop all the sessions
-for (TezSessionState sessionState: TezSessionState.getOpenSessions()) {
+IteratorTezSessionState iter = openSessions.iterator();
+while (iter.hasNext()) {
+ TezSessionState sessionState = iter.next();
if (sessionState.isDefault()) {
sessionState.close(false);
+iter.remove();
}
}
}
@@ -282,5 +294,10 @@ public class TezSessionPoolManager {
}
close(sessionState, keepTmpDir
