Repository: spark Updated Branches: refs/heads/master 7007f72ba -> ff3bea38e
[SPARK-13622][YARN] Issue creating level db for YARN shuffle service ## What changes were proposed in this pull request? This patch will ensure that we trim all path set in yarn.nodemanager.local-dirs and that the the scheme is well removed so the level db can be created. ## How was this patch tested? manual tests. Author: nfraison <[email protected]> Closes #11475 from ashangit/level_db_creation_issue. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ff3bea38 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ff3bea38 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ff3bea38 Branch: refs/heads/master Commit: ff3bea38ed2ac8dac5832f0bf8eac70192a512ef Parents: 7007f72 Author: nfraison <[email protected]> Authored: Mon Mar 28 14:10:25 2016 -0700 Committer: Marcelo Vanzin <[email protected]> Committed: Mon Mar 28 14:10:25 2016 -0700 ---------------------------------------------------------------------- .../org/apache/spark/network/yarn/YarnShuffleService.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/ff3bea38/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java ---------------------------------------------------------------------- diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java index ba6d30a..4bc3c1a 100644 --- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java +++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java @@ -24,6 +24,7 @@ import java.util.List; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.api.*; import org.slf4j.Logger; @@ -118,7 +119,7 @@ public class YarnShuffleService extends AuxiliaryService { // an application was stopped while the NM was down, we expect yarn to call stopApplication() // when it comes back registeredExecutorFile = - findRegisteredExecutorFile(conf.getStrings("yarn.nodemanager.local-dirs")); + findRegisteredExecutorFile(conf.getTrimmedStrings("yarn.nodemanager.local-dirs")); TransportConf transportConf = new TransportConf("shuffle", new HadoopConfigProvider(conf)); // If authentication is enabled, set up the shuffle server to use a @@ -191,12 +192,12 @@ public class YarnShuffleService extends AuxiliaryService { private File findRegisteredExecutorFile(String[] localDirs) { for (String dir: localDirs) { - File f = new File(dir, "registeredExecutors.ldb"); + File f = new File(new Path(dir).toUri().getPath(), "registeredExecutors.ldb"); if (f.exists()) { return f; } } - return new File(localDirs[0], "registeredExecutors.ldb"); + return new File(new Path(localDirs[0]).toUri().getPath(), "registeredExecutors.ldb"); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
