Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 ff04a18f0 -> 69c61fae0
MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread.
Contributed by Xuan Gong
(cherry picked from commit 59a212b6e1265adfa9b55c71b65a22157dfccf77)
Conflicts:
hadoop-mapreduce-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69c61fae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69c61fae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69c61fae
Branch: refs/heads/branch-2.7
Commit: 69c61fae0c7a1501d857430aae6447e518f94a31
Parents: ff04a18
Author: Jason Lowe <[email protected]>
Authored: Mon Feb 1 16:10:56 2016 +0000
Committer: Jason Lowe <[email protected]>
Committed: Mon Feb 1 16:10:56 2016 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 6 +++++
.../org/apache/hadoop/mapred/ClientCache.java | 24 +++++++++++++++++++-
.../hadoop/mapred/ClientServiceDelegate.java | 16 +++++++++++++
.../org/apache/hadoop/mapred/YARNRunner.java | 11 +++++++++
.../mapred/YarnClientProtocolProvider.java | 5 ++--
.../TestYarnClientProtocolProvider.java | 6 +++--
6 files changed, 63 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69c61fae/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index b3f47f7..5a99ff0 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -34,6 +34,9 @@ Release 2.7.3 - UNRELEASED
MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container. (Junping
Du via jianhe)
+ MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread.
+ (Xuan Gong via jlowe)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
@@ -334,6 +337,9 @@ Release 2.6.4 - UNRELEASED
MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in
MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe)
+ MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread.
+ (Xuan Gong via jlowe)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69c61fae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
----------------------------------------------------------------------
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
index 4335c82..93ea5c4 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
@@ -22,11 +22,11 @@ import java.io.IOException;
import java.security.PrivilegedAction;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
@@ -97,4 +97,26 @@ public class ClientCache {
}
});
}
+
+ public void close() throws IOException {
+ if (rm != null) {
+ rm.close();
+ }
+
+ if (hsProxy != null) {
+ RPC.stopProxy(hsProxy);
+ hsProxy = null;
+ }
+
+ if (cache != null && !cache.isEmpty()) {
+ for (ClientServiceDelegate delegate : cache.values()) {
+ if (delegate != null) {
+ delegate.close();
+ delegate = null;
+ }
+ }
+ cache.clear();
+ cache = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69c61fae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index 8517c19..eac8dbc 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -532,4 +533,19 @@ public class ClientServiceDelegate {
throw new IOException("Cannot get log path for a in-progress job");
}
}
+
+ public void close() throws IOException {
+ if (rm != null) {
+ rm.close();
+ }
+
+ if (historyServerProxy != null) {
+ RPC.stopProxy(historyServerProxy);
+ }
+
+ if (realProxy != null) {
+ RPC.stopProxy(realProxy);
+ realProxy = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69c61fae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 3fac655..510f099 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -711,4 +711,15 @@ public class YARNRunner implements ClientProtocol {
envConf + " config settings.");
}
}
+
+ public void close() throws IOException {
+ if (resMgrDelegate != null) {
+ resMgrDelegate.close();
+ resMgrDelegate = null;
+ }
+ if (clientCache != null) {
+ clientCache.close();
+ clientCache = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69c61fae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
----------------------------------------------------------------------
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
index 29d3f66..6583fa2 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
@@ -44,7 +44,8 @@ public class YarnClientProtocolProvider extends
ClientProtocolProvider {
@Override
public void close(ClientProtocol clientProtocol) throws IOException {
- // nothing to do
+ if (clientProtocol instanceof YARNRunner) {
+ ((YARNRunner)clientProtocol).close();
+ }
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69c61fae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
----------------------------------------------------------------------
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
index aeb20cd..4d84fa9 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
@@ -21,10 +21,11 @@ package org.apache.hadoop.mapreduce;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.doNothing;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
@@ -113,6 +114,8 @@ public class TestYarnClientProtocolProvider extends
TestCase {
@Override
protected void serviceStart() throws Exception {
assertTrue(this.client instanceof YarnClientImpl);
+ this.client = spy(this.client);
+ doNothing().when(this.client).close();
((YarnClientImpl) this.client).setRMClient(cRMProtocol);
}
};
@@ -126,5 +129,4 @@ public class TestYarnClientProtocolProvider extends
TestCase {
}
}
}
-
}
