Repository: reef Updated Branches: refs/heads/master 564b88066 -> 5abb8b13a
[REEF-1831] Setting FileSytemUrl for YarnConfiguration used in FileSystem * Inject FileSystemUrl in YarnConfigurationConstructor * Set FileSystemUrl for FileSystem in YarnConfigurationCOnstructor * Bind FileSystemUrl to driver configuration JIRA: [REEF-1831](https://issues.apache.org/jira/browse/REEF-1831) This closes #1332 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/5abb8b13 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/5abb8b13 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/5abb8b13 Branch: refs/heads/master Commit: 5abb8b13a462adff9b8fb1d93d669610857b03b8 Parents: 564b880 Author: Julia Wang <[email protected]> Authored: Thu Jul 13 18:51:09 2017 -0700 Committer: Sergiy Matusevych <[email protected]> Committed: Fri Jul 21 10:01:32 2017 -0700 ---------------------------------------------------------------------- .../bridge/client/YarnJobSubmissionClient.java | 3 ++- .../yarn/driver/YarnContainerManager.java | 11 ++--------- .../yarn/util/YarnConfigurationConstructor.java | 19 +++++++++++++++++-- 3 files changed, 21 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/5abb8b13/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java index f536bd3..fcf03ba 100644 --- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java @@ -277,7 +277,7 @@ public final class YarnJobSubmissionClient { if (!yarnSubmission.getFileSystemUrl().equalsIgnoreCase(FileSystemUrl.DEFAULT_VALUE)) { LOG.log(Level.INFO, "getFileSystemUrl: {0}", yarnSubmission.getFileSystemUrl()); } else { - LOG.log(Level.INFO, "FileSystemUrl is not set"); + LOG.log(Level.INFO, "FileSystemUrl is not set, use default from the environment."); } final List<String> launchCommandPrefix = new ArrayList<String>() {{ @@ -288,6 +288,7 @@ public final class YarnJobSubmissionClient { .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class) .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class) .bindNamedParameter(JobSubmissionDirectoryPrefix.class, yarnSubmission.getJobSubmissionDirectoryPrefix()) + .bindNamedParameter(FileSystemUrl.class, yarnSubmission.getFileSystemUrl()) .bindList(DriverLaunchCommandPrefix.class, launchCommandPrefix) .build(); final YarnJobSubmissionClient client = Tang.Factory.getTang().newInjector(yarnJobSubmissionClientConfig) http://git-wip-us.apache.org/repos/asf/reef/blob/5abb8b13/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java index f89afa0..de8d0b1 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java @@ -43,7 +43,6 @@ import org.apache.reef.runtime.common.driver.resourcemanager.ResourceStatusEvent import org.apache.reef.runtime.common.driver.resourcemanager.RuntimeStatusEventImpl; import org.apache.reef.runtime.common.files.REEFFileNames; import org.apache.reef.runtime.yarn.client.unmanaged.YarnProxyUser; -import org.apache.reef.runtime.yarn.driver.parameters.FileSystemUrl; import org.apache.reef.runtime.yarn.driver.parameters.JobSubmissionDirectory; import org.apache.reef.runtime.yarn.driver.parameters.YarnHeartbeatPeriod; import org.apache.reef.tang.InjectionFuture; @@ -88,9 +87,7 @@ final class YarnContainerManager implements AMRMClientAsync.CallbackHandler, NMC private final DriverStatusManager driverStatusManager; private final String trackingUrl; private final String amRegistrationHost; - private final String jobSubmissionDirectory; - private final String fileSystemUrl; private final REEFFileNames reefFileNames; private final RackNameFormatter rackNameFormatter; private final InjectionFuture<ProgressProvider> progressProvider; @@ -99,7 +96,6 @@ final class YarnContainerManager implements AMRMClientAsync.CallbackHandler, NMC private YarnContainerManager( @Parameter(YarnHeartbeatPeriod.class) final int yarnRMHeartbeatPeriod, @Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory, - @Parameter(FileSystemUrl.class) final String fileSystemUrl, final YarnConfiguration yarnConf, final YarnProxyUser yarnProxyUser, final REEFEventHandlers reefEventHandlers, @@ -130,14 +126,11 @@ final class YarnContainerManager implements AMRMClientAsync.CallbackHandler, NMC this.nodeManager = new NMClientAsyncImpl(this); this.jobSubmissionDirectory = jobSubmissionDirectory; - this.fileSystemUrl = fileSystemUrl; this.reefFileNames = reefFileNames; this.progressProvider = progressProvider; - LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {2}, fileSystemUrl: {3}, " + - "jobSubmissionDirectory: {4}.", - new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.fileSystemUrl, - this.jobSubmissionDirectory}); + LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {3}, jobSubmissionDirectory: {4}.", + new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.jobSubmissionDirectory}); } /** http://git-wip-us.apache.org/repos/asf/reef/blob/5abb8b13/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java index 1f318a7..a54b8e3 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java @@ -19,20 +19,35 @@ package org.apache.reef.runtime.yarn.util; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.reef.runtime.yarn.driver.parameters.FileSystemUrl; import org.apache.reef.tang.ExternalConstructor; +import org.apache.reef.tang.annotations.Parameter; import javax.inject.Inject; +import java.util.logging.Level; +import java.util.logging.Logger; /** * An external constructor that creates YarnConfiguration instances. */ public final class YarnConfigurationConstructor implements ExternalConstructor<YarnConfiguration> { + + private static final Logger LOG = Logger.getLogger(YarnConfigurationConstructor.class.getName()); + + private final String fileSystemUrl; + @Inject - YarnConfigurationConstructor() { + YarnConfigurationConstructor(@Parameter(FileSystemUrl.class) final String fileSystemUrl) { + this.fileSystemUrl = fileSystemUrl; } @Override public YarnConfiguration newInstance() { - return new YarnConfiguration(); + YarnConfiguration yarnConfiguration = new YarnConfiguration(); + if (!fileSystemUrl.equals(FileSystemUrl.DEFAULT_VALUE)) { + yarnConfiguration.set("fs.defaultFS", fileSystemUrl); + LOG.log(Level.INFO, "Set fileSystemUrl in YarnConfigurationConstructor: {0}", fileSystemUrl); + } + return yarnConfiguration; } }
