This is an automated email from the ASF dual-hosted git repository. chengpan pushed a commit to branch branch-1.8 in repository https://gitbox.apache.org/repos/asf/kyuubi.git
commit fa602bfc2cdf3ecce45d954402f347bc4fce27a0 Author: Cheng Pan <[email protected]> AuthorDate: Tue Mar 12 15:22:22 2024 +0800 [KYUUBI #6170] Merge JDBC Trino TPC jobs and Style Check jobs in GitHub Actions workflow We are asked by ASF Infra to limit the max-parallel up to 10 for each workflow. Merge JDBC Trino TPC jobs in GitHub Actions CI to reduce concurrency of workflow, also increase max-parallel of Spark jobs from 1 to 3 Also merge "shell check" job into "style check" job - [ ] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) Pass GA. --- - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6170 from pan3793/ci-2. Closes #6170 9bf505321 [Cheng Pan] merge shell check d0e7ece54 [Cheng Pan] nit b17169573 [Cheng Pan] Tune CI Authored-by: Cheng Pan <[email protected]> Signed-off-by: Cheng Pan <[email protected]> --- .github/workflows/master.yml | 94 +++++++------------------------------------- .github/workflows/style.yml | 5 --- 2 files changed, 14 insertions(+), 85 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index ed1d7cc80..08c9499d5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - max-parallel: 1 + max-parallel: 3 matrix: java: - 8 @@ -339,15 +339,13 @@ jobs: **/target/unit-tests.log **/kyuubi-hive-sql-engine.log* - jdbc-it: - name: JDBC Test + jdbc-trino-tpc-it: + name: JDBC Trino TPC Tests runs-on: ubuntu-22.04 strategy: fail-fast: false - max-parallel: 1 matrix: - java: - - 8 + java: [ 8 ] comment: [ "normal" ] steps: - uses: actions/checkout@v3 @@ -366,11 +364,17 @@ jobs: uses: ./.github/actions/setup-maven - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - - name: Build and test JDBC with maven w/o linters + - name: Build and test JDBC and Trino engines, TPC-H/TPC-DS with maven w/o linters run: | - TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it" - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test + JDBC_TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it" + TRINO_TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it" + TPC_TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch" + ./build/mvn ${MVN_OPT} -pl ${JDBC_TEST_MODULES},${TRINO_TEST_MODULES},${TPC_TEST_MODULES} -am clean install -DskipTests + ./build/mvn ${MVN_OPT} -pl ${JDBC_TEST_MODULES} test + ./build/mvn ${MVN_OPT} -pl ${TRINO_TEST_MODULES} test + ./build/mvn ${MVN_OPT} -pl ${TPC_TEST_MODULES} test \ + -Dmaven.plugin.scalatest.exclude.tags='' \ + -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 @@ -379,78 +383,8 @@ jobs: path: | **/target/unit-tests.log **/kyuubi-jdbc-engine.log* - - trino-it: - name: Trino Test - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - max-parallel: 1 - matrix: - java: - - 8 - comment: [ "normal" ] - steps: - - uses: actions/checkout@v3 - - name: Free up disk space - run: ./.github/scripts/free_disk_space.sh - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - name: Setup JDK ${{ matrix.java }} - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: ${{ matrix.java }} - cache: 'maven' - check-latest: false - - name: Setup Maven - uses: ./.github/actions/setup-maven - - name: Cache Engine Archives - uses: ./.github/actions/cache-engine-archives - - name: Build and test Trino with maven w/o linters - run: | - TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it" - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test - - name: Upload test logs - if: failure() - uses: actions/upload-artifact@v3 - with: - name: unit-tests-log-java-${{ matrix.java }}-trino-${{ matrix.comment }} - path: | - **/target/unit-tests.log **/kyuubi-trino-engine.log* - tpch-tpcds: - name: TPC-H and TPC-DS Tests - runs-on: ubuntu-22.04 - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v3 - - name: Free up disk space - run: ./.github/scripts/free_disk_space.sh - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - name: Setup JDK 8 - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: 8 - cache: 'maven' - check-latest: false - - name: Setup Maven - uses: ./.github/actions/setup-maven - - name: Cache Engine Archives - uses: ./.github/actions/cache-engine-archives - - name: Run TPC-DS Tests - run: | - TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch" - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test \ - -Dmaven.plugin.scalatest.exclude.tags='' \ - -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite - kyuubi-on-k8s-it: name: Kyuubi Server On Kubernetes Integration Test runs-on: ubuntu-22.04 diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 87823ddbd..e4d35538a 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -110,11 +110,6 @@ jobs: echo "you need to manually fix it by following the information told by the command above" echo "---------------------------------------------------------------------------------" - shellcheck: - name: Super Linter and Shellcheck - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - name: Super Linter Checks uses: github/super-linter/slim@v5 env:
