This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 7105f988 [CELEBORN-160][BUILD] Spilt CI workflow (#1107)
7105f988 is described below
commit 7105f988299c393f574a21895a0c3a423cd57c02
Author: Cheng Pan <[email protected]>
AuthorDate: Wed Dec 21 23:47:01 2022 +0800
[CELEBORN-160][BUILD] Spilt CI workflow (#1107)
---
.github/workflows/benchmark.yml | 5 +++-
.github/workflows/benchmark_manual.yml | 5 +++-
.github/workflows/license.yml | 4 ++-
.github/workflows/maven.yml | 46 +++++++++++++++++++++++++++------
.github/workflows/regression.yml | 5 +++-
.github/workflows/regression_manual.yml | 5 +++-
.github/workflows/style.yml | 10 +++----
dev/reformat | 11 +++-----
pom.xml | 9 +++++--
9 files changed, 71 insertions(+), 29 deletions(-)
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 7fb73f29..2d739021 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -33,7 +33,10 @@ jobs:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
- name: Build
- run: ./dev/make-distribution.sh -Pspark-3.2 && rm -rf
/home/hadoop/celeborn-toolkit/dist/* && mv apache-celeborn-*.tgz
/home/hadoop/celeborn-toolkit/dist/
+ run: |
+ dev/make-distribution.sh -Pspark-3.2
+ rm -rf /home/hadoop/celeborn-toolkit/dist/*
+ mv apache-celeborn-*.tgz /home/hadoop/celeborn-toolkit/dist/
- name: Benchmark
run: /home/hadoop/celeborn-toolkit/reg.sh benchmark
diff --git a/.github/workflows/benchmark_manual.yml
b/.github/workflows/benchmark_manual.yml
index dc5dc170..c3e53b65 100644
--- a/.github/workflows/benchmark_manual.yml
+++ b/.github/workflows/benchmark_manual.yml
@@ -33,7 +33,10 @@ jobs:
uses: actions/checkout@v3
- name: Build
- run: ./dev/make-distribution.sh -Pspark-3.2 && rm -rf
/home/hadoop/celeborn-toolkit/dist/* && mv apache-celeborn-*.tgz
/home/hadoop/celeborn-toolkit/dist/
+ run: |
+ dev/make-distribution.sh -Pspark-3.2
+ rm -rf /home/hadoop/celeborn-toolkit/dist/*
+ mv apache-celeborn-*.tgz /home/hadoop/celeborn-toolkit/dist/
- name: Benchmark
run: /home/hadoop/celeborn-toolkit/reg.sh benchmark
diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
index 91780451..098f3046 100644
--- a/.github/workflows/license.yml
+++ b/.github/workflows/license.yml
@@ -42,7 +42,9 @@ jobs:
java-version: 8
cache: 'maven'
check-latest: false
- - run: build/mvn org.apache.rat:apache-rat-plugin:check
-Pgoogle-mirror,spark-3.1
+ - run: |
+ build/mvn org.apache.rat:apache-rat-plugin:check
-Pgoogle-mirror,spark-2.4
+ build/mvn org.apache.rat:apache-rat-plugin:check
-Pgoogle-mirror,spark-3.3
- name: Upload rat report
if: failure()
uses: actions/upload-artifact@v2
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index 18d952a8..5d3660ef 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -10,7 +10,36 @@ on:
branches: [ main ]
jobs:
- build_celeborn_on_spark:
+ service:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ java:
+ - 8
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ cache: maven
+ check-latest: false
+ - name: Test Service with Maven
+ run: build/mvn -Pgoogle-mirror test
+ - name: Upload test log
+ if: failure()
+ uses: actions/upload-artifact@v3
+ with:
+ name: service-unit-test-log
+ path: |
+ **/target/unit-tests.log
+ - name: Upload coverage to Codecov
+ if: matrix.java == 8
+ uses: codecov/codecov-action@v3
+
+ spark:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -33,16 +62,17 @@ jobs:
cache: maven
check-latest: false
- name: Test with Maven
- run: build/mvn -Pgoogle-mirror,spark-${{ matrix.spark }} test
+ run: |
+ SPARK_BINARY_VERSION=${{ matrix.spark }}
+ SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
+ PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
+
TEST_MODULES="client-spark/common,client-spark/spark-${SPARK_MAJOR_VERSION},client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
+ build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
+ build/mvn $PROFILES -pl $TEST_MODULES test
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v3
with:
- name: unit-test-log
+ name: spark-${{ matrix.spark }}-unit-test-log
path: |
**/target/unit-tests.log
- - name: Upload coverage to Codecov
- if: |
- matrix.java == 8 &&
- matrix.spark == '3.3'
- uses: codecov/codecov-action@v3
\ No newline at end of file
diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index c17398f8..29c91d09 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -33,7 +33,10 @@ jobs:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
- name: Build
- run: ./dev/make-distribution.sh -Pspark-3.2 && rm -rf
/home/hadoop/celeborn-toolkit/dist/* && mv apache-celeborn-*.tgz
/home/hadoop/celeborn-toolkit/dist/
+ run: |
+ dev/make-distribution.sh -Pspark-3.2
+ rm -rf /home/hadoop/celeborn-toolkit/dist/*
+ mv apache-celeborn-*.tgz /home/hadoop/celeborn-toolkit/dist/
- name: Regression
run: /home/hadoop/celeborn-toolkit/reg.sh regression
diff --git a/.github/workflows/regression_manual.yml
b/.github/workflows/regression_manual.yml
index f498d8fb..370e7c9a 100644
--- a/.github/workflows/regression_manual.yml
+++ b/.github/workflows/regression_manual.yml
@@ -34,7 +34,10 @@ jobs:
uses: actions/checkout@v3
- name: Build
- run: ./dev/make-distribution.sh -Pspark-3.2 && rm -rf
/home/hadoop/celeborn-toolkit/dist/* && mv apache-celeborn-*.tgz
/home/hadoop/celeborn-toolkit/dist/
+ run: |
+ dev/make-distribution.sh -Pspark-3.2
+ rm -rf /home/hadoop/celeborn-toolkit/dist/*
+ mv apache-celeborn-*.tgz /home/hadoop/celeborn-toolkit/dist/
- name: Regression
run: /home/hadoop/celeborn-toolkit/reg.sh regression
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index c667e060..d5ec23a8 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -33,12 +33,6 @@ jobs:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
- matrix:
- java:
- - 8
- spark:
- - '2.4'
- - '3.1'
steps:
- uses: actions/checkout@v2
- name: Setup JDK 8
@@ -48,4 +42,6 @@ jobs:
java-version: 8
cache: 'maven'
check-latest: false
- - run: build/mvn spotless:check -Pgoogle-mirror,spark-${{ matrix.spark }}
+ - run: |
+ build/mvn spotless:check -Pgoogle-mirror,spark-2.4
+ build/mvn spotless:check -Pgoogle-mirror,spark-3.3
diff --git a/dev/reformat b/dev/reformat
index 486af06c..901dc9ca 100755
--- a/dev/reformat
+++ b/dev/reformat
@@ -18,10 +18,7 @@
set -x
-RSS_HOME="$(cd "`dirname "$0"`/.."; pwd)"
-
-${RSS_HOME}/build/mvn spotless:apply -Pspark-2.4
-
-${RSS_HOME}/build/mvn spotless:apply -Pspark-3.1
-
-${RSS_HOME}/build/mvn spotless:apply -Pflink-1.14
+PROJECT_HOME="$(cd "`dirname "$0"`/.."; pwd)"
+${PROJECT_HOME}/build/mvn spotless:apply -Pflink-1.14
+${PROJECT_HOME}/build/mvn spotless:apply -Pspark-2.4
+${PROJECT_HOME}/build/mvn spotless:apply -Pspark-3.3
diff --git a/pom.xml b/pom.xml
index 9b2bcb2c..c7ffcf1a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,11 +57,15 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>8</java.version>
+ <scala.version>2.12.15</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<maven.version>3.6.3</maven.version>
+ <hadoop.version>3.2.1</hadoop.version>
+ <spark.version>3.3.1</spark.version>
+
<codahale.metrics.version>3.2.6</codahale.metrics.version>
<commons-lang3.version>3.10</commons-lang3.version>
<commons-io.version>2.8.0</commons-io.version>
@@ -73,6 +77,7 @@
<junit.version>4.12</junit.version>
<leveldb.version>1.8</leveldb.version>
<log4j2.version>2.17.2</log4j2.version>
+ <lz4-java.version>1.8.0</lz4-java.version>
<mockito.version>1.10.19</mockito.version>
<mockito-scalatest.version>1.16.37</mockito-scalatest.version>
<netty.version>4.1.77.Final</netty.version>
@@ -83,11 +88,11 @@
<slf4j.version>1.7.36</slf4j.version>
<roaringbitmap.version>0.9.32</roaringbitmap.version>
<snakeyaml.version>1.30</snakeyaml.version>
+ <zstd-jni.version>1.5.2-1</zstd-jni.version>
+
<!-- default flink version -->
<flink.version>1.14.0</flink.version>
- <!-- default hadoop version -->
- <hadoop.version>3.2.1</hadoop.version>
<shading.prefix>org.apache.celeborn.shaded</shading.prefix>
<maven.plugin.antrun.version>3.0.0</maven.plugin.antrun.version>
