LuciferYang commented on code in PR #47378:
URL: https://github.com/apache/spark/pull/47378#discussion_r1680434473
##########
project/SparkBuild.scala:
##########
@@ -674,23 +664,76 @@ object SparkConnectCommon {
// Exclude `scala-library` from assembly.
(assembly / assemblyPackageScala / assembleArtifact) := false,
- // Exclude `pmml-model-*.jar`,
`scala-collection-compat_*.jar`,`jsr305-*.jar` and
- // `netty-*.jar` and `unused-1.0.0.jar` from assembly.
+ // Select jars to exclude from the assembly. This should be consistent
with the content of `maven-shade-plugin`.
(assembly / assemblyExcludedJars) := {
- val cp = (assembly / fullClasspath).value
- cp filter { v =>
- val name = v.data.getName
- name.startsWith("pmml-model-") ||
name.startsWith("scala-collection-compat_") ||
- name.startsWith("jsr305-") || name.startsWith("netty-") || name ==
"unused-1.0.0.jar"
+ val cp = (Runtime / managedClasspath).value
+ val prefixesToShade = Seq(
+ "protobuf-",
+ "grpc-",
+ "proto-google-common-protos-",
+ "guava-",
+ "listenablefuture-",
+ "failureaccess-",
+ "gson-",
+ "perfmark-api-",
+ "jsr305-",
+ "animal-sniffer-annotations-",
+ "annotations-", // Covers both tomcat and android annotations.
+ "j2objc-annotations-",
+ "error_prone_annotations-",
+ "checker-qual-",
+ "unused-")
+ val unshadedJars = filterClasspath(cp, prefixesToShade)
+ val expectedUnshadedPrefixes = Seq(
+ "scala-",
+ "netty-")
+ val unexpectedUnshadedJars = filterClasspath(unshadedJars,
expectedUnshadedPrefixes)
+ if (unexpectedUnshadedJars.nonEmpty) {
Review Comment:
[Run / Run TPC-DS queries with
SF=1](https://github.com/hvanhovell/spark/actions/runs/9966764562/job/27539385165#logs)
```
[error] java.lang.IllegalStateException: Unexpected unshaded jar(s) found:
[error] -
Attributed(/home/runner/.cache/coursier/v1/https/maven-central.storage-download.googleapis.com/maven2/org/jpmml/pmml-model/1.4.8/pmml-model-1.4.8.jar)
[error] at SparkConnectApi$.$anonfun$settings$15(SparkBuild.scala:692)
[error] at scala.Function1.$anonfun$compose$1(Function1.scala:49)
[error] at
sbt.internal.util.$tilde$greater.$anonfun$$u2219$1(TypeFunctions.scala:63)
[error] at sbt.std.Transform$$anon$4.work(Transform.scala:69)
[error] at sbt.Execute.$anonfun$submit$2(Execute.scala:283)
[error] at
sbt.internal.util.ErrorHandling$.wideConvert(ErrorHandling.scala:24)
[error] at sbt.Execute.work(Execute.scala:292)
[error] at sbt.Execute.$anonfun$submit$1(Execute.scala:283)
[error] at
sbt.ConcurrentRestrictions$$anon$4.$anonfun$submitValid$1(ConcurrentRestrictions.scala:265)
[error] at
sbt.CompletionService$$anon$2.call(CompletionService.scala:65)
[error] at
java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
[error] at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
[error] at
java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
[error] at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
[error] at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
[error] at java.base/java.lang.Thread.run(Thread.java:840)
[error] (connect-api / assembly / assemblyExcludedJars)
java.lang.IllegalStateException: Unexpected unshaded jar(s) found:
[error] -
Attributed(/home/runner/.cache/coursier/v1/https/maven-central.storage-download.googleapis.com/maven2/org/jpmml/pmml-model/1.4.8/pmml-model-1.4.8.jar)
[error] Total time: 234 s (03:54), completed Jul 17, 2024, 1:49:40 AM
```
It's a bit strange, only the task `Run TPC-DS queries with SF=1` detected
this issue.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
