Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/303#discussion_r11354818
--- Diff:
yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala ---
@@ -213,48 +214,32 @@ trait ClientBase extends Logging {
val localPath: String = if (_localPath != null) _localPath.trim()
else ""
if (! localPath.isEmpty()) {
val localURI = new URI(localPath)
- val setPermissions = if (destName.equals(ClientBase.APP_JAR)) true
else false
- val destPath = copyRemoteFile(dst, qualifyForLocal(localURI),
replication, setPermissions)
- distCacheMgr.addResource(fs, conf, destPath, localResources,
LocalResourceType.FILE,
- destName, statCache)
+ if (!ClientBase.LOCAL_SCHEME.equals(localURI.getScheme())) {
+ val setPermissions = if (destName.equals(ClientBase.APP_JAR))
true else false
+ val destPath = copyRemoteFile(dst, qualifyForLocal(localURI),
replication, setPermissions)
+ distCacheMgr.addResource(fs, conf, destPath, localResources,
LocalResourceType.FILE,
+ destName, statCache)
+ }
}
}
- // Handle jars local to the ApplicationMaster.
- if ((args.addJars != null) && (!args.addJars.isEmpty())){
- args.addJars.split(',').foreach { case file: String =>
- val localURI = new URI(file.trim())
- val localPath = new Path(localURI)
- val linkname =
Option(localURI.getFragment()).getOrElse(localPath.getName())
- val destPath = copyRemoteFile(dst, localPath, replication)
- // Only add the resource to the Spark ApplicationMaster.
- val appMasterOnly = true
- distCacheMgr.addResource(fs, conf, destPath, localResources,
LocalResourceType.FILE,
- linkname, statCache, appMasterOnly)
- }
- }
-
- // Handle any distributed cache files
- if ((args.files != null) && (!args.files.isEmpty())){
- args.files.split(',').foreach { case file: String =>
- val localURI = new URI(file.trim())
- val localPath = new Path(localURI)
- val linkname =
Option(localURI.getFragment()).getOrElse(localPath.getName())
- val destPath = copyRemoteFile(dst, localPath, replication)
- distCacheMgr.addResource(fs, conf, destPath, localResources,
LocalResourceType.FILE,
- linkname, statCache)
- }
- }
-
- // Handle any distributed cache archives
- if ((args.archives != null) && (!args.archives.isEmpty())) {
- args.archives.split(',').foreach { case file:String =>
- val localURI = new URI(file.trim())
- val localPath = new Path(localURI)
- val linkname =
Option(localURI.getFragment()).getOrElse(localPath.getName())
- val destPath = copyRemoteFile(dst, localPath, replication)
- distCacheMgr.addResource(fs, conf, destPath, localResources,
LocalResourceType.ARCHIVE,
- linkname, statCache)
+ val fileLists = List( (args.addJars, LocalResourceType.FILE, true),
+ (args.files, LocalResourceType.FILE, false),
+ (args.archives, LocalResourceType.ARCHIVE, false) )
+ fileLists.foreach { case (flist, resType, appMasterOnly) =>
+ if (flist != null && !flist.isEmpty()) {
+ flist.split(',').foreach { case file: String =>
+ val localURI = new URI(file.trim())
+ if (!ClientBase.LOCAL_SCHEME.equals(localURI.getScheme())) {
+ val localPath = new Path(localURI)
+ val linkname =
Option(localURI.getFragment()).getOrElse(localPath.getName())
+ val destPath = copyRemoteFile(dst, localPath, replication)
+ // Only add the resource to the Spark ApplicationMaster.
+ val appMasterOnly = true
--- End diff --
Good catch, copy & paste error.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
