[12/24] hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.

2015-07-07 Thread arp 
MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bff67dfe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bff67dfe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bff67dfe

Branch: refs/heads/HDFS-7240
Commit: bff67dfe2f811654ffb1bbcbd87509c185f452b6
Parents: 688617d
Author: Devaraj K deva...@apache.org
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jul 6 13:46:37 2015 +0530

--
 hadoop-mapreduce-project/CHANGES.txt|   3 +
 .../apache/hadoop/mapred/ShuffleHandler.java|   3 +-
 .../hadoop/mapred/TestShuffleHandler.java   | 101 +++
 3 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index 2f80615..2458403 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -542,6 +542,9 @@ Release 2.7.2 - UNRELEASED
 
   BUG FIXES
 
+MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to 
getMapOutputInfo
+if mapId is not in the cache. (zhihai xu via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index eedf42b..ee1be23 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -815,7 +815,8 @@ public class ShuffleHandler extends AuxiliaryService {
 try {
   MapOutputInfo info = mapOutputInfoMap.get(mapId);
   if (info == null) {
-info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+info = getMapOutputInfo(outputBasePathStr + mapId,
+mapId, reduceId, user);
   }
   lastMap =
   sendMapOutput(ctx, ch, user, mapId,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 7053653..746071f 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
   Assert.assertTrue((new String(byteArr)).contains(message));
 } finally {
   shuffleHandler.stop();
+  FileUtil.fullyDelete(absLogDir);
 }
   }
 
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
 conn.disconnect();
 return rc;
   }
+
+  @Test(timeout = 10)
+  public void testGetMapOutputInfo() throws Exception {
+final ArrayListThrowable failures = new ArrayListThrowable(1);
+Configuration conf = new Configuration();
+conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+File absLogDir = new File(target, TestShuffleHandler.class.
+getSimpleName() + LocDir).getAbsoluteFile();
+conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ApplicationId appId = ApplicationId.newInstance(12345, 1);
+String appAttemptId = attempt_12345_1_m_1_0;
+String user = randomUser;
+String reducerId = 0;
+ListFile fileMap = new ArrayListFile();
+

hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.

2015-07-06 Thread devaraj 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 09639ac6b - 415dd6446


MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.

(cherry picked from commit bff67dfe2f811654ffb1bbcbd87509c185f452b6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/415dd644
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/415dd644
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/415dd644

Branch: refs/heads/branch-2
Commit: 415dd64461f480abf5a93c30b21efd0cb38b35a9
Parents: 09639ac
Author: Devaraj K deva...@apache.org
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jul 6 13:47:54 2015 +0530

--
 hadoop-mapreduce-project/CHANGES.txt|   3 +
 .../apache/hadoop/mapred/ShuffleHandler.java|   3 +-
 .../hadoop/mapred/TestShuffleHandler.java   | 101 +++
 3 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/415dd644/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index 7b21dd7..c2bd24c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -275,6 +275,9 @@ Release 2.7.2 - UNRELEASED
 
   BUG FIXES
 
+MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to 
getMapOutputInfo
+if mapId is not in the cache. (zhihai xu via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/415dd644/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index eedf42b..ee1be23 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -815,7 +815,8 @@ public class ShuffleHandler extends AuxiliaryService {
 try {
   MapOutputInfo info = mapOutputInfoMap.get(mapId);
   if (info == null) {
-info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+info = getMapOutputInfo(outputBasePathStr + mapId,
+mapId, reduceId, user);
   }
   lastMap =
   sendMapOutput(ctx, ch, user, mapId,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/415dd644/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 1a0688f..35d9bdd 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
   Assert.assertTrue((new String(byteArr)).contains(message));
 } finally {
   shuffleHandler.stop();
+  FileUtil.fullyDelete(absLogDir);
 }
   }
 
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
 conn.disconnect();
 return rc;
   }
+
+  @Test(timeout = 10)
+  public void testGetMapOutputInfo() throws Exception {
+final ArrayListThrowable failures = new ArrayListThrowable(1);
+Configuration conf = new Configuration();
+conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+File absLogDir = new File(target, TestShuffleHandler.class.
+getSimpleName() + LocDir).getAbsoluteFile();
+conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ApplicationId appId = ApplicationId.newInstance(12345, 1);
+

hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.

2015-07-06 Thread devaraj 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 a6f6ba95e - c9c7a00a8


MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.

(cherry picked from commit bff67dfe2f811654ffb1bbcbd87509c185f452b6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9c7a00a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9c7a00a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9c7a00a

Branch: refs/heads/branch-2.7
Commit: c9c7a00a82016192e9c9a8b15d45f95e7790f820
Parents: a6f6ba9
Author: Devaraj K deva...@apache.org
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jul 6 13:48:44 2015 +0530

--
 hadoop-mapreduce-project/CHANGES.txt|   3 +
 .../apache/hadoop/mapred/ShuffleHandler.java|   3 +-
 .../hadoop/mapred/TestShuffleHandler.java   | 101 +++
 3 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9c7a00a/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index 491d89a..e44624b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,9 @@ Release 2.7.2 - UNRELEASED
 
   BUG FIXES
 
+MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to 
getMapOutputInfo
+if mapId is not in the cache. (zhihai xu via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9c7a00a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 6e069f1..d1a0778 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -806,7 +806,8 @@ public class ShuffleHandler extends AuxiliaryService {
 try {
   MapOutputInfo info = mapOutputInfoMap.get(mapId);
   if (info == null) {
-info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+info = getMapOutputInfo(outputBasePathStr + mapId,
+mapId, reduceId, user);
   }
   lastMap =
   sendMapOutput(ctx, ch, user, mapId,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9c7a00a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 1a0688f..35d9bdd 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
   Assert.assertTrue((new String(byteArr)).contains(message));
 } finally {
   shuffleHandler.stop();
+  FileUtil.fullyDelete(absLogDir);
 }
   }
 
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
 conn.disconnect();
 return rc;
   }
+
+  @Test(timeout = 10)
+  public void testGetMapOutputInfo() throws Exception {
+final ArrayListThrowable failures = new ArrayListThrowable(1);
+Configuration conf = new Configuration();
+conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+File absLogDir = new File(target, TestShuffleHandler.class.
+getSimpleName() + LocDir).getAbsoluteFile();
+conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ApplicationId appId = ApplicationId.newInstance(12345, 1);
+

hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.

2015-07-06 Thread devaraj 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 688617d6d - bff67dfe2


MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bff67dfe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bff67dfe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bff67dfe

Branch: refs/heads/trunk
Commit: bff67dfe2f811654ffb1bbcbd87509c185f452b6
Parents: 688617d
Author: Devaraj K deva...@apache.org
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jul 6 13:46:37 2015 +0530

--
 hadoop-mapreduce-project/CHANGES.txt|   3 +
 .../apache/hadoop/mapred/ShuffleHandler.java|   3 +-
 .../hadoop/mapred/TestShuffleHandler.java   | 101 +++
 3 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index 2f80615..2458403 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -542,6 +542,9 @@ Release 2.7.2 - UNRELEASED
 
   BUG FIXES
 
+MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to 
getMapOutputInfo
+if mapId is not in the cache. (zhihai xu via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index eedf42b..ee1be23 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -815,7 +815,8 @@ public class ShuffleHandler extends AuxiliaryService {
 try {
   MapOutputInfo info = mapOutputInfoMap.get(mapId);
   if (info == null) {
-info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+info = getMapOutputInfo(outputBasePathStr + mapId,
+mapId, reduceId, user);
   }
   lastMap =
   sendMapOutput(ctx, ch, user, mapId,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 7053653..746071f 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
   Assert.assertTrue((new String(byteArr)).contains(message));
 } finally {
   shuffleHandler.stop();
+  FileUtil.fullyDelete(absLogDir);
 }
   }
 
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
 conn.disconnect();
 return rc;
   }
+
+  @Test(timeout = 10)
+  public void testGetMapOutputInfo() throws Exception {
+final ArrayListThrowable failures = new ArrayListThrowable(1);
+Configuration conf = new Configuration();
+conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+File absLogDir = new File(target, TestShuffleHandler.class.
+getSimpleName() + LocDir).getAbsoluteFile();
+conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ApplicationId appId = ApplicationId.newInstance(12345, 1);
+String appAttemptId = attempt_12345_1_m_1_0;
+String user =

[2/7] hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.

2015-07-06 Thread aw 
MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bff67dfe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bff67dfe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bff67dfe

Branch: refs/heads/HADOOP-12111
Commit: bff67dfe2f811654ffb1bbcbd87509c185f452b6
Parents: 688617d
Author: Devaraj K deva...@apache.org
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jul 6 13:46:37 2015 +0530

--
 hadoop-mapreduce-project/CHANGES.txt|   3 +
 .../apache/hadoop/mapred/ShuffleHandler.java|   3 +-
 .../hadoop/mapred/TestShuffleHandler.java   | 101 +++
 3 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index 2f80615..2458403 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -542,6 +542,9 @@ Release 2.7.2 - UNRELEASED
 
   BUG FIXES
 
+MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to 
getMapOutputInfo
+if mapId is not in the cache. (zhihai xu via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index eedf42b..ee1be23 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -815,7 +815,8 @@ public class ShuffleHandler extends AuxiliaryService {
 try {
   MapOutputInfo info = mapOutputInfoMap.get(mapId);
   if (info == null) {
-info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+info = getMapOutputInfo(outputBasePathStr + mapId,
+mapId, reduceId, user);
   }
   lastMap =
   sendMapOutput(ctx, ch, user, mapId,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 7053653..746071f 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
   Assert.assertTrue((new String(byteArr)).contains(message));
 } finally {
   shuffleHandler.stop();
+  FileUtil.fullyDelete(absLogDir);
 }
   }
 
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
 conn.disconnect();
 return rc;
   }
+
+  @Test(timeout = 10)
+  public void testGetMapOutputInfo() throws Exception {
+final ArrayListThrowable failures = new ArrayListThrowable(1);
+Configuration conf = new Configuration();
+conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+File absLogDir = new File(target, TestShuffleHandler.class.
+getSimpleName() + LocDir).getAbsoluteFile();
+conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ApplicationId appId = ApplicationId.newInstance(12345, 1);
+String appAttemptId = attempt_12345_1_m_1_0;
+String user = randomUser;
+String reducerId = 0;
+ListFile fileMap = new ArrayListFile();