This is an automated email from the ASF dual-hosted git repository.
linying pushed a commit to branch dev-2.1.2
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev-2.1.2 by this push:
new cafdad1e8 [Improve] MavenTool resolveArtifacts method improvement
(#3336)
cafdad1e8 is described below
commit cafdad1e807cf1d3025553d34d7c7d26c06dbfcd
Author: benjobs <[email protected]>
AuthorDate: Sun Nov 12 16:26:25 2023 +0800
[Improve] MavenTool resolveArtifacts method improvement (#3336)
* [Improve] MavenTool resolveArtifacts improvement
* minor improvement
* [Improve] MavenTool resolveArtifacts method improvement
---------
Co-authored-by: benjobs <[email protected]>
---
.../streampark/flink/packer/maven/MavenTool.scala | 86 +++++++++++-----------
1 file changed, 44 insertions(+), 42 deletions(-)
diff --git
a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
index a56f41321..c16e1440e 100644
---
a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
+++
b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
@@ -53,7 +53,13 @@ object MavenTool extends Logger {
private[this] lazy val plexusLog = new ConsoleLogger(PlexusLog.LEVEL_INFO,
"streampark-maven")
- private[this] val excludeArtifact = List(
+ /** create maven repository endpoint */
+ private[this] lazy val locator = MavenRepositorySystemUtils.newServiceLocator
+
+ /** default maven local repository */
+ private[this] lazy val localRepo = new
LocalRepository(Workspace.MAVEN_LOCAL_PATH)
+
+ private[this] val excludeArtifact = Set(
Artifact.of("org.apache.flink:force-shading:*"),
Artifact.of("org.apache.flink:flink-shaded-force-shading:*"),
Artifact.of("com.google.code.findbugs:jsr305:*"),
@@ -198,48 +204,51 @@ object MavenTool extends Logger {
}
val (repoSystem, session) = getMavenEndpoint()
+
val artifacts = mavenArtifacts.map(
- e => {
- new DefaultArtifact(e.groupId, e.artifactId, e.classifier, "jar",
e.version) -> e.extensions
- })
+ e => new DefaultArtifact(e.groupId, e.artifactId, e.classifier, "jar",
e.version))
- logInfo(s"start resolving dependencies: ${artifacts.mkString}")
+ val exclusions = mavenArtifacts
+ .flatMap(_.extensions.map(_.split(":")))
+ .map(a => Artifact(a.head, a.last, null)) ++ excludeArtifact
val remoteRepos = getRemoteRepos()
- // read relevant artifact descriptor info
- // plz don't simplify the following lambda syntax to maintain the
readability of the code.
- val dependenciesArtifacts = artifacts
- .map(artifact => new ArtifactDescriptorRequest(artifact._1, remoteRepos,
null) -> artifact._2)
- .map(descReq => repoSystem.readArtifactDescriptor(session, descReq._1)
-> descReq._2)
- .flatMap(
- result =>
- result._1.getDependencies
- .filter(
- dep => {
- dep.getScope match {
- case "compile" if
!excludeArtifact.exists(_.filter(dep.getArtifact)) =>
- val ga =
s"${dep.getArtifact.getGroupId}:${dep.getArtifact.getArtifactId}"
- val exclusion = result._2.contains(ga)
- if (exclusion) {
- val art = result._1.getArtifact
- val name = s"${art.getGroupId}:${art.getArtifactId}"
- logInfo(s"[MavenTool] $name dependencies exclusion $ga")
- }
- !exclusion
- case _ => false
- }
- })
- .map(_.getArtifact))
-
- val mergedArtifacts = artifacts.map(_._1) ++ dependenciesArtifacts
- logInfo(s"resolved dependencies: ${mergedArtifacts.mkString}")
+ // read relevant artifact descriptor info and excluding items if necessary.
+ val dependencies = artifacts
+ .map(artifact => new ArtifactDescriptorRequest(artifact, remoteRepos,
null))
+ .map(descReq => repoSystem.readArtifactDescriptor(session, descReq))
+ .flatMap(_.getDependencies)
+ .filter(_.getScope == "compile")
+ .filter(dep => !exclusions.exists(_.filter(dep.getArtifact)))
+ .map(_.getArtifact)
+
+ val mergedArtifacts = artifacts ++ dependencies
+
+ logInfo(
+ s"""
+ |start resolving dependencies...
+
|--------------------------------------------------------------------------------
+ ||User-declared dependencies list：
+ |${artifacts.mkString(",\n")}
+ |
+ ||Indirect dependencies list:
+ |${dependencies.mkString(",\n")}
+ |
+ ||Exclusion indirect dependencies list:
+ |${exclusions.map(x =>
s"${x.groupId}:${x.artifactId}").mkString(",\n")}
+ |
+ ||Final dependencies list:
+ |${mergedArtifacts.mkString(",\n")}
+
|--------------------------------------------------------------------------------
+ |""".stripMargin
+ )
// download artifacts
- val artReqs =
- mergedArtifacts.map(artifact => new ArtifactRequest(artifact,
remoteRepos, null))
+ val artReq = mergedArtifacts.map(a => new ArtifactRequest(a, remoteRepos,
null))
+
repoSystem
- .resolveArtifacts(session, artReqs)
+ .resolveArtifacts(session, artReq)
.map(_.getArtifact.getFile)
.toSet
}
@@ -247,13 +256,6 @@ object MavenTool extends Logger {
/** create composite maven endpoint */
private[this] def getMavenEndpoint(): (RepositorySystem,
RepositorySystemSession) = {
- /** create maven repository endpoint */
-
- lazy val locator = MavenRepositorySystemUtils.newServiceLocator
-
- /** default maven local repository */
- lazy val localRepo = new LocalRepository(Workspace.MAVEN_LOCAL_PATH)
-
def newRepoSystem(): RepositorySystem = {
locator.addService(
classOf[RepositoryConnectorFactory],
