xushiyan commented on code in PR #7511:
URL: https://github.com/apache/hudi/pull/7511#discussion_r1090152941
##########
.github/workflows/bot.yml:
##########
@@ -13,94 +13,23 @@ env:
MVN_ARGS: -ntp -B -V -Pwarn-log
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn
jobs:
- test-spark:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- include:
- - scalaProfile: "scala-2.11"
- sparkProfile: "spark2.4"
-
- - scalaProfile: "scala-2.12"
- sparkProfile: "spark2.4"
-
- - scalaProfile: "scala-2.12"
- sparkProfile: "spark3.1"
-
- - scalaProfile: "scala-2.12"
- sparkProfile: "spark3.2"
-
- - scalaProfile: "scala-2.12"
- sparkProfile: "spark3.3"
-
- steps:
- - uses: actions/checkout@v2
- - name: Set up JDK 8
- uses: actions/setup-java@v2
- with:
- java-version: '8'
- distribution: 'adopt'
- architecture: x64
- - name: Build Project
- env:
- SCALA_PROFILE: ${{ matrix.scalaProfile }}
- SPARK_PROFILE: ${{ matrix.sparkProfile }}
- run:
- mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl
hudi-examples/hudi-examples-spark,hudi-spark-datasource/hudi-spark -am
-DskipTests=true $MVN_ARGS
- - name: Quickstart Test
- env:
- SCALA_PROFILE: ${{ matrix.scalaProfile }}
- SPARK_PROFILE: ${{ matrix.sparkProfile }}
- run:
- mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl
hudi-examples/hudi-examples-spark $MVN_ARGS
- - name: UT - Common & Spark
- env:
- SCALA_PROFILE: ${{ matrix.scalaProfile }}
- SPARK_PROFILE: ${{ matrix.sparkProfile }}
- if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4
as it's covered by Azure CI
- run:
- mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl
hudi-common,hudi-spark-datasource/hudi-spark $MVN_ARGS
-
- test-flink:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- include:
- - flinkProfile: "flink1.13"
- - flinkProfile: "flink1.14"
- - flinkProfile: "flink1.15"
- steps:
- - uses: actions/checkout@v2
- - name: Set up JDK 8
- uses: actions/setup-java@v2
- with:
- java-version: '8'
- distribution: 'adopt'
- architecture: x64
- - name: Build Project
- env:
- SCALA_PROFILE: 'scala-2.12'
- FLINK_PROFILE: ${{ matrix.flinkProfile }}
- run:
- mvn clean install -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl
hudi-examples/hudi-examples-flink -am -Davro.version=1.10.0 -DskipTests=true
$MVN_ARGS
- - name: Quickstart Test
- env:
- SCALA_PROFILE: 'scala-2.12'
- FLINK_PROFILE: ${{ matrix.flinkProfile }}
- run:
- mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl
hudi-examples/hudi-examples-flink $MVN_ARGS
-
validate-bundles:
runs-on: ubuntu-latest
strategy:
matrix:
include:
- - flinkProfile: 'flink1.15'
- sparkProfile: 'spark3.3'
- - flinkProfile: 'flink1.14'
- sparkProfile: 'spark3.2'
+ - flinkProfile: 'flink1.13'
+ sparkProfile: 'spark'
+ - flinkProfile: 'flink1.13'
+ sparkProfile: 'spark2.4'
- flinkProfile: 'flink1.13'
sparkProfile: 'spark3.1'
+ - flinkProfile: 'flink1.14'
+ sparkProfile: 'spark3.2'
+ - flinkProfile: 'flink1.15'
+ sparkProfile: 'spark3.3'
+ - flinkProfile: 'flink1.15'
+ sparkProfile: 'spark3'
Review Comment:
variate this to control `SPARK_PROFILE` which controls what bundles to
download for testing in packaging/bundle-validation/ci_run.sh
##########
packaging/bundle-validation/ci_run.sh:
##########
@@ -27,10 +27,29 @@
# This is to run by GitHub Actions CI tasks from the project root directory
# and it contains the CI environment-specific variables.
-HUDI_VERSION=$1
+HUDI_VERSION=0.12.2-rc1
+REPO_BASE_URL=https://repository.apache.org/content/repositories/orgapachehudi-1106/org/apache/hudi
Review Comment:
we set these 2 vars to download hudi bundle jars directly from apache's
artifactory. `REPO_BASE_URL` should be changed accordingly wrt `HUDI_VERSION`,
which can be any release candidate. These info is available in the RC voting
email
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
