Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/4754#discussion_r25406707
--- Diff: core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala ---
@@ -251,24 +251,17 @@ object SparkSubmit {
}
val isYarnCluster = clusterManager == YARN && deployMode == CLUSTER
-
- // Resolve maven dependencies if there are any and add classpath to
jars. Add them to py-files
- // too for packages that include Python code
- val resolvedMavenCoordinates =
- SparkSubmitUtils.resolveMavenCoordinates(
- args.packages, Option(args.repositories), Option(args.ivyRepoPath))
- if (!resolvedMavenCoordinates.trim.isEmpty) {
- if (args.jars == null || args.jars.trim.isEmpty) {
- args.jars = resolvedMavenCoordinates
- } else {
- args.jars += s",$resolvedMavenCoordinates"
- }
- if (args.isPython) {
- if (args.pyFiles == null || args.pyFiles.trim.isEmpty) {
- args.pyFiles = resolvedMavenCoordinates
- } else {
- args.pyFiles += s",$resolvedMavenCoordinates"
- }
+ if (args.packagesResolved != null) {
+ args.jars = mergeFileLists(args.jars, args.packagesResolved)
+ if (args.isPython) args.pyFiles = mergeFileLists(args.pyFiles,
args.packagesResolved)
+ } else {
+ // This part needs to be added in case SparkSubmitDriverBootstrapper
is not called
+ val resolvedMavenCoordinates =
+ SparkSubmitUtils.resolveMavenCoordinates(
+ args.packages, Option(args.repositories),
Option(args.ivyRepoPath)).mkString(",")
+ if (resolvedMavenCoordinates.trim.length > 0) {
+ args.jars = mergeFileLists(args.jars, resolvedMavenCoordinates)
+ if (args.isPython) args.pyFiles = mergeFileLists(args.pyFiles,
resolvedMavenCoordinates)
}
--- End diff --
Looks like there is an opportunity for abstracting the two cases here:
```
val packagesResolved =
if (args.packagesResolved != null) {
args.packagesResolved
} else {
SparkSubmitUtils.resolveMavenCoordinates(...).mkString(",")
}
if (packagesResolved.nonEmpty) {
args.jars = mergeFileLists(args.jars, packagesResolved)
if (args.isPython) {
args.pyFiles = mergeFileLists(args.pyFiles, packagesResolved)
}
}
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
