HeartSaVioR commented on a change in pull request #21339: [SPARK-24287][Core]
Spark -packages option should support classifier, no-transitive, and custom conf
URL: https://github.com/apache/spark/pull/21339#discussion_r268539299
##########
File path: core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
##########
@@ -1098,20 +1147,51 @@ private[spark] object SparkSubmitUtils {
cacheDirectory: File): String = {
artifacts.map { artifactInfo =>
val artifact = artifactInfo.asInstanceOf[Artifact].getModuleRevisionId
+ val classifier =
artifactInfo.asInstanceOf[Artifact].getId.getExtraAttribute("classifier")
+ val suffix = if (classifier == null) "" else s"-${classifier}"
cacheDirectory.getAbsolutePath + File.separator +
-
s"${artifact.getOrganisation}_${artifact.getName}-${artifact.getRevision}.jar"
+
s"${artifact.getOrganisation}_${artifact.getName}-${artifact.getRevision}${suffix}.jar"
}.mkString(",")
}
- /** Adds the given maven coordinates to Ivy's module descriptor. */
+ /** Adds the given artifact coordinates to Ivy's module descriptor. */
def addDependenciesToIvy(
md: DefaultModuleDescriptor,
artifacts: Seq[MavenCoordinate],
+ ivySettings: IvySettings,
ivyConfName: String): Unit = {
- artifacts.foreach { mvn =>
- val ri = ModuleRevisionId.newInstance(mvn.groupId, mvn.artifactId,
mvn.version)
- val dd = new DefaultDependencyDescriptor(ri, false, false)
+ artifacts.foreach { art =>
+ val ri = ModuleRevisionId.newInstance(art.groupId, art.artifactId,
art.version)
+ val dd = art.extraParams.get("transitive") match {
+ case Some(t) => new DefaultDependencyDescriptor(null, ri, false,
false, t.toBoolean)
+ case None => new DefaultDependencyDescriptor(ri, false, false)
+ }
dd.addDependencyConfiguration(ivyConfName, ivyConfName + "(runtime)")
+
+ art.extraParams.foreach { case (param, pvalue) =>
+ param match {
+ // Exclude dependencies(name separated by #) for this artifact
+ case "exclude" => pvalue.split("#").foreach { ex =>
+ dd.addExcludeRule(ivyConfName,
+ createExclusion("*:*" + ex + "*:*", ivySettings, ivyConfName))}
Review comment:
For now, if we want to exclude `org.json:json` we should give `json` as
parameter and it will exclude everything which artifact name contains `json`.
This behaves as very misleading: could we just let end users specify all for
groupId and artifactId, and let end users explicitly use `*` if really needed?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
