Repository: reef Updated Branches: refs/heads/master 205df7bf1 -> a5395f875
[REEF-1666] Better logging in YARN job submission * Print YARN application ID with higher log level * Log IOException on copyFromLocalFile * Minor refactoring for readability JIRA: [REEF-1666](https://issues.apache.org/jira/browse/REEF-1666) Pull Request: This closes #1180 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/a5395f87 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/a5395f87 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/a5395f87 Branch: refs/heads/master Commit: a5395f8758436b92ea0ca0c3590624ef5ef0fc46 Parents: 205df7b Author: Sergiy Matusevych <[email protected]> Authored: Mon Nov 7 15:56:13 2016 -0800 Committer: Markus Weimer <[email protected]> Committed: Thu Nov 10 09:34:11 2016 -0800 ---------------------------------------------------------------------- .../yarn/client/YarnSubmissionHelper.java | 8 ++--- .../runtime/yarn/client/uploader/JobFolder.java | 34 +++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/a5395f87/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnSubmissionHelper.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnSubmissionHelper.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnSubmissionHelper.java index 3eb92f4..b4841ac 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnSubmissionHelper.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnSubmissionHelper.java @@ -81,14 +81,14 @@ public final class YarnSubmissionHelper implements Closeable{ LOG.log(Level.FINE, "Requesting Application ID from YARN."); this.yarnClientApplication = this.yarnClient.createApplication(); - this.applicationResponse = yarnClientApplication.getNewApplicationResponse(); - this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext(); - this.applicationId = applicationSubmissionContext.getApplicationId(); + this.applicationResponse = this.yarnClientApplication.getNewApplicationResponse(); + this.applicationSubmissionContext = this.yarnClientApplication.getApplicationSubmissionContext(); + this.applicationId = this.applicationSubmissionContext.getApplicationId(); this.tokenProvider = tokenProvider; this.commandPrefixList = commandPrefixList; this.launcherClazz = REEFLauncher.class; this.configurationFilePaths = Collections.singletonList(this.fileNames.getDriverConfigurationPath()); - LOG.log(Level.FINEST, "YARN Application ID: {0}", applicationId); + LOG.log(Level.INFO, "YARN Application ID: {0}", this.applicationId); } public YarnSubmissionHelper(final YarnConfiguration yarnConfiguration, http://git-wip-us.apache.org/repos/asf/reef/blob/a5395f87/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/uploader/JobFolder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/uploader/JobFolder.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/uploader/JobFolder.java index 6de0561..f9a61ba 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/uploader/JobFolder.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/uploader/JobFolder.java @@ -57,53 +57,57 @@ public final class JobFolder { } /** - * Uploads the given file to the DFS. - * - * @param localFile + * Upload given file to the DFS. + * @param localFile File on local FS to upload to the (remote) DFS. * @return the Path representing the file on the DFS. - * @throws IOException + * @throws IOException if unable to upload the file or local file cannot be read. */ public Path upload(final File localFile) throws IOException { + if (!localFile.exists()) { throw new FileNotFoundException(localFile.getAbsolutePath()); } final Path source = new Path(localFile.getAbsolutePath()); final Path destination = new Path(this.path, localFile.getName()); + try { this.fileSystem.copyFromLocalFile(source, destination); - } catch (final IOException e) { - LOG.log(Level.SEVERE, "Unable to upload {0} to {1}", new Object[]{source, destination}); - throw e; + } catch (final IOException ex) { + LOG.log(Level.SEVERE, "Unable to upload " + source + " to " + destination, ex); + throw ex; } - LOG.log(Level.FINE, "Uploaded {0} to {1}", new Object[]{source, destination}); + + LOG.log(Level.FINE, "Uploaded {0} to {1}", new Object[] {source, destination}); return destination; } /** * Shortcut to first upload the file and then form a LocalResource for the YARN submission. - * - * @param localFile - * @return - * @throws IOException + * @param localFile File on local FS to upload to the (remote) DFS. + * @return an instance of a LocalResource on (remote) DFS. + * @throws IOException if unable to upload the file or local file cannot be read. */ public LocalResource uploadAsLocalResource(final File localFile, final LocalResourceType type) throws IOException { - final Path p = upload(localFile); - return getLocalResourceForPath(p, type); + final Path remoteFile = upload(localFile); + return getLocalResourceForPath(remoteFile, type); } /** * Creates a LocalResource instance for the JAR file referenced by the given Path. */ public LocalResource getLocalResourceForPath(final Path jarPath, final LocalResourceType type) throws IOException { - final LocalResource localResource = Records.newRecord(LocalResource.class); + final FileStatus status = FileContext.getFileContext(fileSystem.getUri()).getFileStatus(jarPath); + + final LocalResource localResource = Records.newRecord(LocalResource.class); localResource.setType(type); localResource.setVisibility(LocalResourceVisibility.APPLICATION); localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath())); localResource.setTimestamp(status.getModificationTime()); localResource.setSize(status.getLen()); + return localResource; }
