This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 6ba4b7e13 [CELEBORN-850][INFRA] Add SBT CI
6ba4b7e13 is described below
commit 6ba4b7e138a96cc50f53153bed1439b9caf227cf
Author: Fu Chen <[email protected]>
AuthorDate: Tue Aug 1 18:14:58 2023 +0800
[CELEBORN-850][INFRA] Add SBT CI
### What changes were proposed in this pull request?
This PR adds new GitHub Actions workflows to enable Continuous Integration
using SBT based on #1764
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GA
Closes #1771 from cfmcgrady/sbt-ci.
Authored-by: Fu Chen <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/sbt.yml | 163 ++++++++++++++++++++++++++++++++++++++++++++
project/CelebornBuild.scala | 34 +++++++--
2 files changed, 192 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/sbt.yml b/.github/workflows/sbt.yml
new file mode 100644
index 000000000..5f35361b3
--- /dev/null
+++ b/.github/workflows/sbt.yml
@@ -0,0 +1,163 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+name: Celeborn SBT CI
+
+on:
+ push:
+ branches:
+ - main
+ - branch-*
+ pull_request:
+ branches:
+ - main
+ - branch-*
+
+jobs:
+ service:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ java:
+ - 8
+ - 11
+ - 17
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ cache: maven
+ check-latest: false
+ - name: Test Service with SBT
+ run: |
+ build/sbt "clean; test"
+ - name: Upload test log
+ if: failure()
+ uses: actions/upload-artifact@v3
+ with:
+ name: service-${{ matrix.java }}-unit-test-log
+ path: |
+ **/target/test-reports/**
+
+ spark2:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ java:
+ - 8
+ spark:
+ - '2.4'
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ cache: maven
+ check-latest: false
+ - name: Test with SBT
+ run: |
+ build/sbt -Pspark-${{ matrix.spark }} "clean;
celeborn-spark-group/test"
+ - name: Upload test log
+ if: failure()
+ uses: actions/upload-artifact@v3
+ with:
+ name: spark-${{ matrix.spark }}-unit-test-log
+ path: |
+ **/target/test-reports/**
+
+ spark3:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ java:
+ - 8
+ - 11
+ - 17
+ spark:
+ - '3.0'
+ - '3.1'
+ - '3.2'
+ - '3.3'
+ - '3.4'
+ exclude:
+ # SPARK-33772: Spark supports JDK 17 since 3.3.0
+ - java: 17
+ spark: '3.0'
+ - java: 17
+ spark: '3.1'
+ - java: 17
+ spark: '3.2'
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ cache: maven
+ check-latest: false
+ - name: Test with SBT
+ run: |
+ build/sbt -Pspark-${{ matrix.spark }} "clean;
celeborn-spark-group/test"
+ - name: Upload test log
+ if: failure()
+ uses: actions/upload-artifact@v3
+ with:
+ name: spark-${{ matrix.spark }}-unit-test-log
+ path: |
+ **/target/test-reports/**
+
+ flink:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ java:
+ - 8
+ - 11
+ flink:
+ - '1.14'
+ - '1.15'
+ - '1.17'
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ cache: maven
+ check-latest: false
+ - name: Test with SBT
+ run: |
+ build/sbt -Pflink-${{ matrix.flink }} "clean;
celeborn-flink-group/test"
+ - name: Upload test log
+ if: failure()
+ uses: actions/upload-artifact@v3
+ with:
+ name: flink-${{ matrix.flink }}-unit-test-log
+ path: |
+ **/target/test-reports/**
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index f55aa26f6..fbcbbf3a2 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -89,7 +89,23 @@ object CelebornCommonSettings {
// Make sure any tests in any project that uses Spark is configured for
running well locally
Test / javaOptions ++= Seq(
- "-Xmx4g"
+ "-Xmx4g",
+ "-XX:+IgnoreUnrecognizedVMOptions",
+ "--add-opens=java.base/java.lang=ALL-UNNAMED",
+ "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
+ "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED",
+ "--add-opens=java.base/java.io=ALL-UNNAMED",
+ "--add-opens=java.base/java.net=ALL-UNNAMED",
+ "--add-opens=java.base/java.nio=ALL-UNNAMED",
+ "--add-opens=java.base/java.util=ALL-UNNAMED",
+ "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
+ "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
+ "--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED",
+ "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
+ "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
+ "--add-opens=java.base/sun.security.action=ALL-UNNAMED",
+ "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
+ "-Dio.netty.tryReflectionSetAccessible=true"
),
testOptions += Tests.Argument("-oF"),
@@ -125,7 +141,7 @@ object CelebornCommonSettings {
lazy val commonUnitTestDependencies = Seq(
"org.mockito" % "mockito-core" % "4.11.0" % "test",
"org.scalatest" %% "scalatest" % "3.2.16" % "test",
- "junit" % "junit" % "4.12" % "test",
+ "junit" % "junit" % "4.13.2" % "test",
// https://www.scala-sbt.org/1.x/docs/Testing.html
"com.github.sbt" % "junit-interface" % "0.13.3" % "test")
}
@@ -489,7 +505,11 @@ trait SparkClientProjects {
val sparkVersion: String
val zstdJniVersion: String
- def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt,
sparkClientShade)
+ def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt,
sparkGroup, sparkClientShade)
+
+ // for test only, don't use this group for any other projects
+ lazy val sparkGroup = (project withId "celeborn-spark-group")
+ .aggregate(sparkCommon, sparkClient, sparkIt)
def sparkCommon: Project = {
Project("celeborn-spark-common", file("client-spark/common"))
@@ -553,7 +573,7 @@ trait SparkClientProjects {
) ++ commonUnitTestDependencies
)
}
-
+
def sparkClientShade: Project = {
Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
.dependsOn(sparkClient)
@@ -657,7 +677,11 @@ trait FlinkClientProjects {
lazy val flinkClientsDependency: ModuleID = "org.apache.flink" %
"flink-clients" % flinkVersion % "test"
lazy val flinkRuntimeWebDependency: ModuleID = "org.apache.flink" %
"flink-runtime-web" % flinkVersion % "test"
- def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt,
flinkClientShade)
+ def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt,
flinkGroup, flinkClientShade)
+
+ // for test only, don't use this group for any other projects
+ lazy val flinkGroup = (project withId "celeborn-flink-group")
+ .aggregate(flinkCommon, flinkClient, flinkIt)
// get flink major version. e.g:
// 1.17.0 -> 1.17
