Repository: reef Updated Branches: refs/heads/master 4392ab83b -> 4e4e568fa
[REEF-1669]Removing YarnClient from YarnContainerManager Currently we have both YarnClient and resourceManager (AMRMClientAsync) in YarnContainerManager. We call init() and start() for both of them. But inside driver, we are not able to get correct YarnConfiguration for YarnClient, that makes yarnClient.getNodeReports() fail to connect to RM. In YarnContainerManager, we already have onNodesUpdated() which gives the list of the NodeReports on the fly. yarnClient.getNodeReports() was called at beginning and it only gives a static list. So looks like we don't need to call it anyway. Based on RM folks, REEF should not use YarnClient at driver side. This PR is to remove YarnClient from YarnContainerManager. It also added some logs for environment variables with a lower log level or debug. JIRA: [REEF-1669](https://issues.apache.org/jira/browse/REEF-1669) This closes #1182 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/4e4e568f Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/4e4e568f Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/4e4e568f Branch: refs/heads/master Commit: 4e4e568faa3c3d625433b17bb4d15e0cd7897d7a Parents: 4392ab8 Author: Julia Wang <[email protected]> Authored: Wed Nov 16 11:29:05 2016 -0800 Committer: Sergiy Matusevych <[email protected]> Committed: Thu Nov 17 17:59:21 2016 -0800 ---------------------------------------------------------------------- .../reef/runtime/yarn/YarnClasspathProvider.java | 11 +++++++++++ .../runtime/yarn/driver/YarnContainerManager.java | 18 ------------------ 2 files changed, 11 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/4e4e568f/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java index 8b400de..ad3685e 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/YarnClasspathProvider.java @@ -27,6 +27,7 @@ import org.apache.reef.util.Optional; import javax.inject.Inject; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -59,6 +60,7 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider { HADOOP_HDFS_HOME + "/lib/*", HADOOP_MAPRED_HOME + "/*", HADOOP_MAPRED_HOME + "/lib/*", + HADOOP_COMMON_HOME + "/etc/hadoop/client/*", HADOOP_HOME + "/etc/hadoop", HADOOP_HOME + "/share/hadoop/common/*", HADOOP_HOME + "/share/hadoop/common/lib/*", @@ -74,6 +76,7 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider { @Inject YarnClasspathProvider(final YarnConfiguration yarnConfiguration) { + logEnvVariable(); boolean needsLegacyClasspath = false; // will be set to true below whenever we encounter issues with the YARN Configuration final ClassPathBuilder builder = new ClassPathBuilder(); @@ -153,6 +156,14 @@ public final class YarnClasspathProvider implements RuntimeClasspathProvider { return this.classPathSuffix; } + private static void logEnvVariable() { + if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) { + for (final Map.Entry<String, String> entry : System.getenv().entrySet()) { + LOG.log(CLASSPATH_LOG_LEVEL, "Environment variable: Key: {0}, Value: {1}.", + new Object[]{entry.getKey(), entry.getValue()}); + } + } + } private void logClasspath() { if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) { http://git-wip-us.apache.org/repos/asf/reef/blob/4e4e568f/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java index 58fcca1..41be1e8 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; @@ -63,7 +62,6 @@ final class YarnContainerManager private static final String RUNTIME_NAME = "YARN"; - private final YarnClient yarnClient = YarnClient.createYarnClient(); private final Queue<AMRMClient.ContainerRequest> requestsBeforeSentToRM = new ConcurrentLinkedQueue<>(); private final Queue<AMRMClient.ContainerRequest> requestsAfterSentToRM = new ConcurrentLinkedQueue<>(); private final Map<String, String> nodeIdToRackName = new ConcurrentHashMap<>(); @@ -107,8 +105,6 @@ final class YarnContainerManager this.trackingURLProvider = trackingURLProvider; this.rackNameFormatter = rackNameFormatter; - this.yarnClient.init(this.yarnConf); - this.resourceManager = AMRMClientAsync.createAMRMClientAsync(yarnRMHeartbeatPeriod, this); this.nodeManager = new NMClientAsyncImpl(this); @@ -303,9 +299,6 @@ final class YarnContainerManager * This method is called from DriverRuntimeStartHandler via YARNRuntimeStartHandler. */ void onStart() { - - this.yarnClient.start(); - this.resourceManager.init(this.yarnConf); this.resourceManager.start(); @@ -313,16 +306,6 @@ final class YarnContainerManager this.nodeManager.start(); try { - for (final NodeReport nodeReport : this.yarnClient.getNodeReports(NodeState.RUNNING)) { - onNodeReport(nodeReport); - } - } catch (IOException | YarnException e) { - LOG.log(Level.WARNING, "Unable to fetch node reports from YARN.", e); - onRuntimeError(e); - } - - try { - this.registration.setRegistration( this.resourceManager.registerApplicationMaster("", 0, this.trackingURLProvider.getTrackingUrl())); @@ -334,7 +317,6 @@ final class YarnContainerManager try (final FSDataOutputStream out = fs.create(outputFileName)) { out.writeBytes(this.trackingURLProvider.getTrackingUrl() + '\n'); } - } catch (final YarnException | IOException e) { LOG.log(Level.WARNING, "Unable to register application master.", e); onRuntimeError(e);
