turboFei commented on code in PR #3282:
URL: https://github.com/apache/celeborn/pull/3282#discussion_r2108075447
##########
.github/workflows/maven.yml:
##########
@@ -146,6 +146,43 @@ jobs:
**/target/test-reports/**
**/target/unit-tests.log
+ spark4:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ java:
+ - 17
+ spark:
+ - '4.0'
+ shuffle-plugin-class:
+ - 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ cache: maven
+ check-latest: false
+ - name: Test with Maven
+ run: |
+ SPARK_BINARY_VERSION=${{ matrix.spark }}
+ SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
+ PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
+
TEST_MODULES="client-spark/common,client-spark/spark-3,client-spark/spark-3-columnar-common,client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
+ build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
+# build/mvn $PROFILES -pl $TEST_MODULES
-Dspark.shuffle.sort.io.plugin.class=${{ matrix.shuffle-plugin-class }} test
Review Comment:
Due to dependency conflicts.
I have excluded all the RESTful dependecies from spark-core and spark-sql
```
<!-- exclude RESTful service dependencies -->
<exclusion>
<groupId>jakarta.servlet</groupId>
<artifactId>jakarta.servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
```
But still meet below issue:
```
A needed class was not found. This could be due to an error in your runpath.
Missing class: jakarta/ws/rs/core/Configuration
java.lang.NoClassDefFoundError: jakarta/ws/rs/core/Configuration
at java.base/java.lang.ClassLoader.defineClass1(Native Method)
at java.base/java.lang.ClassLoader.defineClass(ClassLoader.java:1017)
at
java.base/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
at
java.base/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
at
java.base/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:760)
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
at
java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
at java.base/java.lang.ClassLoader.defineClass1(Native Method)
at java.base/java.lang.ClassLoader.defineClass(ClassLoader.java:1017)
at
java.base/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
at
java.base/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
at
java.base/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:760)
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
at
java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
at java.base/java.lang.ClassLoader.defineClass1(Native Method)
at java.base/java.lang.ClassLoader.defineClass(ClassLoader.java:1017)
at
java.base/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
at
java.base/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
at
java.base/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:760)
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
at
java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
at
org.apache.celeborn.server.common.HttpService.startInternal(HttpService.scala:336)
at
org.apache.celeborn.server.common.HttpService.startHttpServer(HttpService.scala:210)
at
org.apache.celeborn.service.deploy.MiniClusterFeature.createMaster(MiniClusterFeature.scala:141)
at
org.apache.celeborn.service.deploy.MiniClusterFeature.setUpMaster(MiniClusterFeature.scala:180)
at
org.apache.celeborn.service.deploy.MiniClusterFeature.setUpMaster$(MiniClusterFeature.scala:179)
at
org.apache.celeborn.tests.spark.CelebornSortSuite.setUpMaster(CelebornSortSuite.scala:30)
at
org.apache.celeborn.service.deploy.MiniClusterFeature.setUpMiniCluster(MiniClusterFeature.scala:278)
at
org.apache.celeborn.service.deploy.MiniClusterFeature.setupMiniClusterWithRandomPorts(MiniClusterFeature.scala:103)
at
org.apache.celeborn.service.deploy.MiniClusterFeature.setupMiniClusterWithRandomPorts$(MiniClusterFeature.scala:75)
at
org.apache.celeborn.tests.spark.CelebornSortSuite.setupMiniClusterWithRandomPorts(CelebornSortSuite.scala:30)
at
org.apache.celeborn.tests.spark.SparkTestBase.beforeAll(SparkTestBase.scala:54)
at
org.apache.celeborn.tests.spark.SparkTestBase.beforeAll$(SparkTestBase.scala:52)
at
org.apache.celeborn.tests.spark.CelebornSortSuite.beforeAll(CelebornSortSuite.scala:30)
at
org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:212)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at
org.apache.celeborn.tests.spark.CelebornSortSuite.run(CelebornSortSuite.scala:30)
at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:47)
at
org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13(Runner.scala:1321)
at
org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13$adapted(Runner.scala:1315)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1315)
at
org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:992)
at
org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:970)
at
org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1481)
at
org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:970)
at org.scalatest.tools.Runner$.run(Runner.scala:798)
at org.scalatest.tools.Runner.run(Runner.scala)
at
org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2or3(ScalaTestRunner.java:43)
at
org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:26)
Caused by: java.lang.ClassNotFoundException: jakarta.ws.rs.core.Configuration
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)
at
java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
... 57 more
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
