YARN-9067. YARN Resource Manager is running OOM because of leak of Configuration Object. Contributed by Eric Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efc4d91c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efc4d91c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efc4d91c Branch: refs/heads/HDFS-12943 Commit: efc4d91cbeab8a13f6d61cb0e56443adb2d77559 Parents: fe7dab8 Author: Weiwei Yang <w...@apache.org> Authored: Thu Nov 29 09:34:14 2018 +0800 Committer: Weiwei Yang <w...@apache.org> Committed: Thu Nov 29 09:34:14 2018 +0800 ---------------------------------------------------------------------- .../hadoop/yarn/service/webapp/ApiServer.java | 209 +++++++++++-------- .../hadoop/yarn/service/ServiceClientTest.java | 2 +- .../yarn/service/client/ServiceClient.java | 1 + 3 files changed, 126 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index db831ba..88aeefd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -118,10 +118,13 @@ public class ApiServer { @Override public Void run() throws YarnException, IOException { ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - sc.actionBuild(service); - sc.close(); + try { + sc.init(YARN_CONFIG); + sc.start(); + sc.actionBuild(service); + } finally { + sc.close(); + } return null; } }); @@ -133,11 +136,14 @@ public class ApiServer { @Override public ApplicationId run() throws IOException, YarnException { ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - ApplicationId applicationId = sc.actionCreate(service); - sc.close(); - return applicationId; + try { + sc.init(YARN_CONFIG); + sc.start(); + ApplicationId applicationId = sc.actionCreate(service); + return applicationId; + } finally { + sc.close(); + } } }); serviceStatus.setDiagnostics("Application ID: " + applicationId); @@ -245,29 +251,32 @@ public class ApiServer { public Integer run() throws Exception { int result = 0; ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - Exception stopException = null; try { - result = sc.actionStop(appName, destroy); - if (result == EXIT_SUCCESS) { - LOG.info("Successfully stopped service {}", appName); - } - } catch (Exception e) { - LOG.info("Got exception stopping service", e); - stopException = e; - } - if (destroy) { - result = sc.actionDestroy(appName); - if (result == EXIT_SUCCESS) { - LOG.info("Successfully deleted service {}", appName); + sc.init(YARN_CONFIG); + sc.start(); + Exception stopException = null; + try { + result = sc.actionStop(appName, destroy); + if (result == EXIT_SUCCESS) { + LOG.info("Successfully stopped service {}", appName); + } + } catch (Exception e) { + LOG.info("Got exception stopping service", e); + stopException = e; } - } else { - if (stopException != null) { - throw stopException; + if (destroy) { + result = sc.actionDestroy(appName); + if (result == EXIT_SUCCESS) { + LOG.info("Successfully deleted service {}", appName); + } + } else { + if (stopException != null) { + throw stopException; + } } + } finally { + sc.close(); } - sc.close(); return result; } }); @@ -378,13 +387,16 @@ public class ApiServer { @Override public Map<String, Long> run() throws YarnException, IOException { ServiceClient sc = new ServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - Map<String, Long> original = sc.flexByRestService(appName, - Collections.singletonMap(componentName, - component.getNumberOfContainers())); - sc.close(); - return original; + try { + sc.init(YARN_CONFIG); + sc.start(); + Map<String, Long> original = sc.flexByRestService(appName, + Collections.singletonMap(componentName, + component.getNumberOfContainers())); + return original; + } finally { + sc.close(); + } } }); ServiceStatus status = new ServiceStatus(); @@ -632,12 +644,15 @@ public class ApiServer { public Integer run() throws YarnException, IOException { int result = 0; ServiceClient sc = new ServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - result = sc - .actionFlex(appName, componentCountStrings); - sc.close(); - return Integer.valueOf(result); + try { + sc.init(YARN_CONFIG); + sc.start(); + result = sc + .actionFlex(appName, componentCountStrings); + return Integer.valueOf(result); + } finally { + sc.close(); + } } }); if (result == EXIT_SUCCESS) { @@ -658,12 +673,15 @@ public class ApiServer { @Override public String run() throws YarnException, IOException { ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - String newLifeTime = sc.updateLifetime(appName, - updateAppData.getLifetime()); - sc.close(); - return newLifeTime; + try { + sc.init(YARN_CONFIG); + sc.start(); + String newLifeTime = sc.updateLifetime(appName, + updateAppData.getLifetime()); + return newLifeTime; + } finally { + sc.close(); + } } }); ServiceStatus status = new ServiceStatus(); @@ -681,11 +699,14 @@ public class ApiServer { @Override public ApplicationId run() throws YarnException, IOException { ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - ApplicationId appId = sc.actionStartAndGetId(appName); - sc.close(); - return appId; + try { + sc.init(YARN_CONFIG); + sc.start(); + ApplicationId appId = sc.actionStartAndGetId(appName); + return appId; + } finally { + sc.close(); + } } }); LOG.info("Successfully started service " + appName); @@ -702,14 +723,17 @@ public class ApiServer { ServiceStatus status = new ServiceStatus(); ugi.doAs((PrivilegedExceptionAction<Void>) () -> { ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) { - sc.actionUpgradeExpress(service); - } else { - sc.initiateUpgrade(service); + try { + sc.init(YARN_CONFIG); + sc.start(); + if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) { + sc.actionUpgradeExpress(service); + } else { + sc.initiateUpgrade(service); + } + } finally { + sc.close(); } - sc.close(); return null; }); LOG.info("Service {} version {} upgrade initialized", service.getName(), @@ -724,11 +748,14 @@ public class ApiServer { final UserGroupInformation ugi) throws IOException, InterruptedException { int result = ugi.doAs((PrivilegedExceptionAction<Integer>) () -> { ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - int exitCode = sc.actionCancelUpgrade(serviceName); - sc.close(); - return exitCode; + try { + sc.init(YARN_CONFIG); + sc.start(); + int exitCode = sc.actionCancelUpgrade(serviceName); + return exitCode; + } finally { + sc.close(); + } }); if (result == EXIT_SUCCESS) { ServiceStatus status = new ServiceStatus(); @@ -793,10 +820,13 @@ public class ApiServer { return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> { int result1; ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - result1 = sc.actionUpgrade(service, containers); - sc.close(); + try { + sc.init(YARN_CONFIG); + sc.start(); + result1 = sc.actionUpgrade(service, containers); + } finally { + sc.close(); + } return result1; }); } @@ -815,12 +845,15 @@ public class ApiServer { public Integer run() throws YarnException, IOException { int result = 0; ServiceClient sc = new ServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - result = sc - .actionDecommissionInstances(appName, instances); - sc.close(); - return Integer.valueOf(result); + try { + sc.init(YARN_CONFIG); + sc.start(); + result = sc + .actionDecommissionInstances(appName, instances); + return Integer.valueOf(result); + } finally { + sc.close(); + } } }); if (result == EXIT_SUCCESS) { @@ -840,11 +873,14 @@ public class ApiServer { return ugi.doAs((PrivilegedExceptionAction<Service>) () -> { ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - Service app1 = sc.getStatus(serviceName); - sc.close(); - return app1; + try { + sc.init(YARN_CONFIG); + sc.start(); + Service app1 = sc.getStatus(serviceName); + return app1; + } finally { + sc.close(); + } }); } @@ -855,12 +891,15 @@ public class ApiServer { return ugi.doAs((PrivilegedExceptionAction<ComponentContainers[]>) () -> { ComponentContainers[] result; ServiceClient sc = getServiceClient(); - sc.init(YARN_CONFIG); - sc.start(); - result = sc.getContainers(serviceName, componentNames, version, - containerStates); - sc.close(); - return result; + try { + sc.init(YARN_CONFIG); + sc.start(); + result = sc.getContainers(serviceName, componentNames, version, + containerStates); + return result; + } finally { + sc.close(); + } }); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java index d022614..89366b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java @@ -75,7 +75,7 @@ public class ServiceClientTest extends ServiceClient { public void forceStop() { expectedInstances.clear(); - super.stop(); + stop(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 713d890..5cdb8c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -163,6 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, if (registryClient != null) { registryClient.stop(); } + fs.getFileSystem().close(); super.serviceStop(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
