(incubator-streampark) branch job-state updated: [Improve] resolve maven artifacts improvement

[benjobs]

This is an automated email from the ASF dual-hosted git repository.

benjobs pushed a commit to branch job-state
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git


The following commit(s) were added to refs/heads/job-state by this push:
     new 8f3fc72bc [Improve] resolve maven artifacts improvement
8f3fc72bc is described below

commit 8f3fc72bc58eb76b61b6a65be3ef4e5362aef772
Author: benjobs <benj...@gmail.com>
AuthorDate: Fri Nov 10 22:41:24 2023 +0800

    [Improve] resolve maven artifacts improvement
---
 .../console/core/service/impl/AppBuildPipeServiceImpl.java  | 13 ++++++++++++-
 .../org/apache/streampark/flink/packer/maven/Artifact.scala |  2 ++
 .../apache/streampark/flink/packer/maven/MavenTool.scala    | 13 +++++++------
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git 
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
 
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
index b60112d32..f978e570e 100644
--- 
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
+++ 
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/AppBuildPipeServiceImpl.java
@@ -95,6 +95,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -424,7 +425,17 @@ public class AppBuildPipeServiceImpl
 
           // 3). pom dependency
           if (!app.getDependencyInfo().mavenArts().isEmpty()) {
-            
jars.addAll(MavenTool.resolveArtifactsAsJava(app.getDependencyInfo().mavenArts()));
+            Set<File> mavenArts =
+                MavenTool.resolveArtifactsAsJava(
+                    app.getDependencyInfo()
+                        .mavenArts()
+                        .filter(
+                            x -> {
+                              File jarFile = new File(localUploadDIR, 
x.jarName());
+                              return !jarFile.exists();
+                            })
+                        .toSet());
+            jars.addAll(mavenArts);
           }
 
           // 4). local uploadDIR to hdfs uploadsDIR
diff --git 
a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/Artifact.scala
 
b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/Artifact.scala
index cff3c0398..6bcf71fa0 100644
--- 
a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/Artifact.scala
+++ 
b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/Artifact.scala
@@ -38,6 +38,8 @@ case class Artifact(
     }
   }
 
+  lazy val jarName = s"$artifactId-$version.jar"
+
 }
 
 object Artifact {
diff --git 
a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
 
b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
index 05562a870..ed59af118 100644
--- 
a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
+++ 
b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/maven/MavenTool.scala
@@ -43,7 +43,6 @@ import javax.annotation.{Nonnull, Nullable}
 
 import java.io.File
 import java.util
-import java.util.{HashSet, Set => JavaSet}
 
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
@@ -56,8 +55,10 @@ object MavenTool extends Logger {
 
   private[this] val excludeArtifact = List(
     Artifact.of("org.apache.flink:force-shading:*"),
+    Artifact.of("org.apache.flink:flink-shaded-force-shading:*"),
     Artifact.of("com.google.code.findbugs:jsr305:*"),
-    Artifact.of("org.apache.logging.log4j:*:*"))
+    Artifact.of("org.apache.logging.log4j:*:*")
+  )
 
   private[this] def getRemoteRepos(): List[RemoteRepository] = {
     val builder =
@@ -145,7 +146,7 @@ object MavenTool extends Logger {
       }
       req.setResourceTransformers(transformer.toList)
       // issue: https://github.com/apache/incubator-streampark/issues/2350
-      req.setFilters(List(new ShadeFilter))
+      req.setFilters(List(new ShadedFilter))
       req.setRelocators(Lists.newArrayList())
       req
     }
@@ -178,7 +179,7 @@ object MavenTool extends Logger {
     buildFatJar(mainClass, jarLibs ++ artFilePaths, outFatJarPath)
   }
 
-  def resolveArtifactsAsJava(mavenArtifacts: Set[Artifact]): JavaSet[File] = 
resolveArtifacts(
+  def resolveArtifactsAsJava(mavenArtifacts: Set[Artifact]): util.Set[File] = 
resolveArtifacts(
     mavenArtifacts).asJava
 
   /**
@@ -256,13 +257,13 @@ object MavenTool extends Logger {
     (repoSystem, session)
   }
 
-  class ShadeFilter extends Filter {
+  private class ShadedFilter extends Filter {
     override def canFilter(jar: File): Boolean = true
 
     override def isFiltered(name: String): Boolean = {
       if (name.startsWith("META-INF/")) {
         if (name.endsWith(".SF") || name.endsWith(".DSA") || 
name.endsWith(".RSA")) {
-          logInfo(s"shade ignore file: $name")
+          logInfo(s"shaded ignore file: $name")
           return true
         }
       }