[1/2] spark git commit: Revert [HOT FIX] [YARN] Check whether `/lib` exists before listing its files

2014-12-04 Thread andrewor14 
Repository: spark
Updated Branches:
  refs/heads/master ca379039f - fd8525334


Revert [HOT FIX] [YARN] Check whether `/lib` exists before listing its files

This reverts commit 90ec643e9af4c8bbb9000edca08c07afb17939c7.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/87437df0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/87437df0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/87437df0

Branch: refs/heads/master
Commit: 87437df036305c6f467dc6e0bf7891d27d266a6b
Parents: ca37903
Author: Andrew Or and...@databricks.com
Authored: Thu Dec 4 21:53:38 2014 -0800
Committer: Andrew Or and...@databricks.com
Committed: Thu Dec 4 21:53:38 2014 -0800

--
 .../apache/spark/deploy/yarn/ClientBase.scala   | 27 +---
 1 file changed, 12 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/87437df0/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git 
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala 
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 290d994..8e4360e 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -243,21 +243,18 @@ private[spark] trait ClientBase extends Logging {
   val libsURI = new URI(libsDir)
   val jarLinks = ListBuffer.empty[String]
   if (libsURI.getScheme != LOCAL_SCHEME) {
-val localPath = getQualifiedLocalPath(libsURI)
-val localFs = FileSystem.get(localPath.toUri, hadoopConf)
-if (localFs.exists(localPath)) {
-  val jars = localFs.listFiles(localPath, /* recursive */ false)
-  while (jars.hasNext) {
-val jar = jars.next()
-val name = jar.getPath.getName
-if (name.startsWith(datanucleus-)) {
-  // copy to remote and add to classpath
-  val src = jar.getPath
-  val destPath = copyFileToRemote(dst, src, replication)
-  distCacheMgr.addResource(localFs, hadoopConf, destPath,
-localResources, LocalResourceType.FILE, name, statCache)
-  jarLinks += name
-}
+val localURI = getQualifiedLocalPath(libsURI).toUri()
+val jars = FileSystem.get(localURI, hadoopConf).listFiles(new 
Path(localURI.getPath), false)
+while (jars.hasNext) {
+  val jar = jars.next()
+  val name = jar.getPath.getName
+  if (name.startsWith(datanucleus-)) {
+// copy to remote and add to classpath
+val src = jar.getPath
+val destPath = copyFileToRemote(dst, src, replication)
+distCacheMgr.addResource(fs, hadoopConf, destPath,
+  localResources, LocalResourceType.FILE, name, statCache)
+jarLinks += name
   }
 }
   } else {


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

[1/2] spark git commit: Revert [HOT FIX] [YARN] Check whether `/lib` exists before listing its files

2014-12-04 Thread andrewor14 
Repository: spark
Updated Branches:
  refs/heads/branch-1.2 6c4363178 - a8d8077dc


Revert [HOT FIX] [YARN] Check whether `/lib` exists before listing its files

This reverts commit 38cb2c3a36a5c9ead4494cbc3dde008c2f0698ce.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/325babe8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/325babe8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/325babe8

Branch: refs/heads/branch-1.2
Commit: 325babe8a3c1ab8cc10cc7cee5b6a53757774154
Parents: 6c43631
Author: Andrew Or and...@databricks.com
Authored: Thu Dec 4 21:54:37 2014 -0800
Committer: Andrew Or and...@databricks.com
Committed: Thu Dec 4 21:54:37 2014 -0800

--
 .../apache/spark/deploy/yarn/ClientBase.scala   | 27 +---
 1 file changed, 12 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/325babe8/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git 
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala 
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 290d994..8e4360e 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -243,21 +243,18 @@ private[spark] trait ClientBase extends Logging {
   val libsURI = new URI(libsDir)
   val jarLinks = ListBuffer.empty[String]
   if (libsURI.getScheme != LOCAL_SCHEME) {
-val localPath = getQualifiedLocalPath(libsURI)
-val localFs = FileSystem.get(localPath.toUri, hadoopConf)
-if (localFs.exists(localPath)) {
-  val jars = localFs.listFiles(localPath, /* recursive */ false)
-  while (jars.hasNext) {
-val jar = jars.next()
-val name = jar.getPath.getName
-if (name.startsWith(datanucleus-)) {
-  // copy to remote and add to classpath
-  val src = jar.getPath
-  val destPath = copyFileToRemote(dst, src, replication)
-  distCacheMgr.addResource(localFs, hadoopConf, destPath,
-localResources, LocalResourceType.FILE, name, statCache)
-  jarLinks += name
-}
+val localURI = getQualifiedLocalPath(libsURI).toUri()
+val jars = FileSystem.get(localURI, hadoopConf).listFiles(new 
Path(localURI.getPath), false)
+while (jars.hasNext) {
+  val jar = jars.next()
+  val name = jar.getPath.getName
+  if (name.startsWith(datanucleus-)) {
+// copy to remote and add to classpath
+val src = jar.getPath
+val destPath = copyFileToRemote(dst, src, replication)
+distCacheMgr.addResource(fs, hadoopConf, destPath,
+  localResources, LocalResourceType.FILE, name, statCache)
+jarLinks += name
   }
 }
   } else {


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org