Author: jdere
Date: Thu Sep 4 18:23:22 2014
New Revision: 1622523
URL: http://svn.apache.org/r1622523
Log:
HIVE-5857: Reduce tasks do not work in uber mode in YARN (Adam Kawa, reviewed
by Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/queries/clientpositive/uber_reduce.q
hive/trunk/ql/src/test/results/clientpositive/uber_reduce.q.out
Modified:
hive/trunk/itests/src/test/resources/testconfiguration.properties
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1622523&r1=1622522&r2=1622523&view=diff
==============================================================================
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Thu Sep
4 18:23:22 2014
@@ -44,6 +44,7 @@ minimr.query.files=auto_sortmerge_join_1
stats_counter_partitioned.q,\
temp_table_external.q,\
truncate_column_buckets.q,\
+ uber_reduce.q,\
udf_using.q
minitez.query.files.shared=alter_merge_2_orc.q,\
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1622523&r1=1622522&r2=1622523&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Thu
Sep 4 18:23:22 2014
@@ -326,7 +326,9 @@ public final class Utilities {
assert path != null;
if (!gWorkMap.containsKey(path)) {
Path localPath;
- if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
+ if (conf.getBoolean("mapreduce.task.uberized", false) &&
name.equals(REDUCE_PLAN_NAME)) {
+ localPath = new Path(name);
+ } else if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
localPath = path;
} else {
localPath = new Path(name);
Added: hive/trunk/ql/src/test/queries/clientpositive/uber_reduce.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/uber_reduce.q?rev=1622523&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/uber_reduce.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/uber_reduce.q Thu Sep 4
18:23:22 2014
@@ -0,0 +1,11 @@
+SET mapreduce.job.ubertask.enable=true;
+SET mapreduce.job.ubertask.maxreduces=1;
+SET mapred.reduce.tasks=1;
+
+-- Uberized mode is a YARN option, ignore this test for non-YARN Hadoop
versions
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20,0.20S)
+
+CREATE TABLE T1(key STRING, val STRING);
+LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1;
+
+SELECT count(*) FROM T1;
Added: hive/trunk/ql/src/test/results/clientpositive/uber_reduce.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/uber_reduce.q.out?rev=1622523&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/uber_reduce.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/uber_reduce.q.out Thu Sep 4
18:23:22 2014
@@ -0,0 +1,31 @@
+PREHOOK: query: -- Uberized mode is a YARN option, ignore this test for
non-YARN Hadoop versions
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20,0.20S)
+
+CREATE TABLE T1(key STRING, val STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@T1
+POSTHOOK: query: -- Uberized mode is a YARN option, ignore this test for
non-YARN Hadoop versions
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20,0.20S)
+
+CREATE TABLE T1(key STRING, val STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@T1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@t1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@t1
+PREHOOK: query: SELECT count(*) FROM T1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*) FROM T1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+6
