This is an automated email from the ASF dual-hosted git repository.
jiangmaolin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 54dae051510 Add dynamic matrix workflows (#32337)
54dae051510 is described below
commit 54dae05151080892258a24b5c51259058f540bc1
Author: 孙念君 Nianjun Sun <[email protected]>
AuthorDate: Wed Jul 31 10:30:26 2024 +0800
Add dynamic matrix workflows (#32337)
* Add : add dynamic ci in nightly CI
* Refactor : remove the os default value
* Refactor : remove the os default value
* Refactor : add new file to trigger the dynamic matrix
* Refactor : add new file to trigger the dynamic matrix
* Refactor : revert the nighlty-ci file
* Refactor : remove some useless condition
* Refactor : add dynamic workflow for nighlyt sql e2e
* Refactor : make the mandatory condition as required
* Refactor : remove useless inputs
* Refactor : remove useless condition
---
.github/workflows/nightly-ci-dynamic.yml | 140 ++++++++++++++++++++
.../workflows/nightly-e2e-operation-dynamic.yml | 89 +++++++++++++
.github/workflows/nightly-e2e-sql-dynamic.yml | 144 +++++++++++++++++++++
3 files changed, 373 insertions(+)
diff --git a/.github/workflows/nightly-ci-dynamic.yml
b/.github/workflows/nightly-ci-dynamic.yml
new file mode 100644
index 00000000000..1bb3e6da904
--- /dev/null
+++ b/.github/workflows/nightly-ci-dynamic.yml
@@ -0,0 +1,140 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Nightly - CI Dynamic
+
+on:
+ workflow_dispatch:
+ inputs:
+ java-version:
+ description: 'JDK Version'
+ required: false
+ default: "[21]"
+ os:
+ description: 'System OS'
+ required: false
+ default: "['ubuntu-latest']"
+ job:
+ description: 'Job Name'
+ required: true
+ default: "ci-dynamic"
+
+env:
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
+
+jobs:
+ global-environment:
+ name: Import Global Environment
+ uses: ./.github/workflows/required-reusable.yml
+
+ ci-dynamic:
+ if: github.repository == 'apache/shardingsphere' &&
github.event.inputs.job == 'ci-dynamic'
+ name: CI - JDK ${{ matrix.java-version }} on ${{ matrix.os }}
+ needs: global-environment
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 90
+ strategy:
+ max-parallel: 20
+ fail-fast: false
+ matrix:
+ os: ${{ fromJson(github.event.inputs.os) }}
+ java-version: ${{ fromJson(github.event.inputs.java-version) }}
+ steps:
+ - name: Support Long Paths in Windows
+ if: matrix.os == 'windows-latest'
+ run: git config --global core.longpaths true
+ - uses: actions/checkout@v4
+ - uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: ${{ matrix.java-version }}
+ - uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-cache-${{ github.sha }}
+ restore-keys: |
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-cache-
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-
+ - name: Build Project with Maven
+ run: ./mvnw clean install -B -ntp -T1C
+ - name: Build Examples with Maven
+ run: ./mvnw clean package -B -f examples/pom.xml -T1C
+
+ ci-jdk8-dynamic:
+ if: github.repository == 'apache/shardingsphere' &&
github.event.inputs.job == 'ci-jdk8-dynamic'
+ name: CI - JDK 8 on ${{ matrix.os }}
+ needs: global-environment
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 90
+ strategy:
+ max-parallel: 20
+ fail-fast: false
+ matrix:
+ os: ${{ fromJson(github.event.inputs.os) }}
+ steps:
+ - name: Support Long Paths in Windows
+ if: matrix.os == 'windows-latest'
+ run: git config --global core.longpaths true
+ - uses: actions/checkout@v4
+ - uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: 11
+ - uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-cache-${{ github.sha }}
+ restore-keys: |
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-cache-
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-
+ - name: Build prod with Maven
+ run: ./mvnw clean install -DskipTests -B -ntp -T1C
+ - name: Setup JDK 8 for Test
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'zulu'
+ java-version: 8
+ - name: Run Tests with JDK 8
+ run: ./mvnw test -B -ntp -fae -T1C
+
+ ci-native-test:
+ if: github.repository == 'apache/shardingsphere' &&
github.event.inputs.job == 'ci-native-test'
+ name: NativeTest CI - GraalVM CE on ${{ matrix.os }}
+ needs: global-environment
+ runs-on: ${{ fromJson(github.event.inputs.os) }}
+ timeout-minutes: 90
+ strategy:
+ max-parallel: 20
+ fail-fast: false
+ matrix:
+ os: ${{ fromJson(github.event.inputs.os) }}
+ steps:
+ - uses: actions/checkout@v4
+ - uses: graalvm/setup-graalvm@v1
+ with:
+ java-version: '22.0.2'
+ distribution: 'graalvm-community'
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ - uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-cache-${{ github.sha }}
+ restore-keys: |
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-cache-
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-
+ - name: Run nativeTest with GraalVM CE
+ run: ./mvnw -PnativeTestInShardingSphere -T1C -e clean test
diff --git a/.github/workflows/nightly-e2e-operation-dynamic.yml
b/.github/workflows/nightly-e2e-operation-dynamic.yml
new file mode 100644
index 00000000000..e231e0a4392
--- /dev/null
+++ b/.github/workflows/nightly-e2e-operation-dynamic.yml
@@ -0,0 +1,89 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Nightly - E2E Operation Dynamic
+
+on:
+ workflow_dispatch:
+ inputs:
+ java-version:
+ description: 'java-version'
+ required: true
+ default: "[ 11, 21 ]"
+ operation:
+ description: 'operation'
+ required: true
+ default: "[ transaction, pipeline, showprocesslist ]"
+ image:
+ description: 'image'
+ required: true
+ default: "[ { type: 'it.docker.mysql.version', version:
'mysql:5.7,mysql:8.0' } ]"
+
+concurrency:
+ group: e2e-pipeline-${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3 -Dspotless.apply.skip=true
-Dfailsafe.skipAfterFailureCount=1 -Dio.netty.leakDetectionLevel=advanced
+
+jobs:
+ global-environment:
+ name: Import Global Environment
+ uses: ./.github/workflows/required-reusable.yml
+
+ e2e-operation-job:
+ if: ${{ needs.global-environment.outputs.GLOBAL_IS_NIGHTLY_JOB_EXECUTABLE
== 'true' }}
+ name: E2E - ${{ matrix.operation }} on ${{ matrix.image.version }}
+ needs: global-environment
+ runs-on: ${{ needs.global-environment.outputs.GLOBAL_RUNS_ON }}
+ strategy:
+ max-parallel: 20
+ fail-fast: false
+ matrix:
+ java-version: [ 11, 21 ]
+ operation: [ transaction, pipeline, showprocesslist ]
+ image: [ { type: "it.docker.mysql.version", version:
"mysql:5.7,mysql:8.0" }, { type: "it.docker.postgresql.version", version:
"postgres:10-alpine,postgres:11-alpine,postgres:12-alpine,postgres:13-alpine,postgres:14-alpine"
}, { type: "it.docker.opengauss.version", version:
"enmotech/opengauss:2.1.0,enmotech/opengauss:3.0.0" }, { type:
"it.docker.mariadb.version", version: "mariadb:11" } ]
+ exclude:
+ - operation: transaction
+ image: { type: "it.docker.mariadb.version", version: "mariadb:11" }
+ - operation: showprocesslist
+ image: { type: "it.docker.postgresql.version", version:
"postgres:10-alpine,postgres:11-alpine,postgres:12-alpine,postgres:13-alpine,postgres:14-alpine"
}
+ - operation: showprocesslist
+ image: { type: "it.docker.opengauss.version", version:
"enmotech/opengauss:2.1.0,enmotech/opengauss:3.0.0" }
+ - operation: showprocesslist
+ image: { type: "it.docker.mariadb.version", version: "mariadb:11" }
+ timeout-minutes: 40
+ steps:
+ - name: Checkout Project
+ uses: actions/checkout@v4
+ - name: Retrieve Maven Caches
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-e2e-cache-${{ github.sha }}
+ restore-keys: |
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-e2e-cache-
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-
+ - name: Setup JDK ${{ matrix.java-version }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: ${{ matrix.java-version }}
+ - name: Build ${{ matrix.operation }} E2E Image
+ run: ./mvnw -B clean install -am -pl test/e2e/operation/${{
matrix.operation }} -Pit.env.docker -DskipTests
+ - name: Run ${{ matrix.operation }} on ${{ matrix.image.version }}
+ run: ./mvnw -nsu -B install -f test/e2e/operation/${{ matrix.operation
}}/pom.xml -D${{ matrix.operation }}.it.env.type=docker -D${{ matrix.operation
}}.${{ matrix.image.type }}=${{ matrix.image.version }}
diff --git a/.github/workflows/nightly-e2e-sql-dynamic.yml
b/.github/workflows/nightly-e2e-sql-dynamic.yml
new file mode 100644
index 00000000000..9798431ec92
--- /dev/null
+++ b/.github/workflows/nightly-e2e-sql-dynamic.yml
@@ -0,0 +1,144 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Nightly - E2E SQL Dynamic
+
+on:
+ workflow_dispatch:
+ workflow_dispatch:
+ inputs:
+ adapter:
+ description: 'adapter'
+ required: true
+ default: "[ proxy, jdbc ]"
+ mode:
+ description: 'mode'
+ required: true
+ default: "[ Standalone, Cluster ]"
+ database:
+ description: 'database'
+ required: true
+ default: "[ MySQL, PostgreSQL, openGauss ]"
+ scenario:
+ description: 'scenario'
+ required: true
+ default: "[ empty_rules ]"
+ additional-options:
+ description: 'additional-options'
+ required: false
+ default: "[ '' ]"
+
+concurrency:
+ group: e2e-${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3 -Dspotless.apply.skip=true
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1
+
+jobs:
+ global-environment:
+ name: Import Global Environment
+ uses: ./.github/workflows/required-reusable.yml
+
+ build-e2e-image:
+ name: Build E2E Image
+ needs: global-environment
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: 21
+ - uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-e2e-cache-${{ github.sha }}
+ restore-keys: |
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-e2e-cache-
+ ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-
+ - name: Build E2E Image
+ run: ./mvnw -B clean install -am -pl test/e2e/sql -Pit.env.docker
-DskipTests -Dspotless.apply.skip=true
+ - name: Save E2E Image
+ run: docker save -o /tmp/${{
needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-proxy-test.tar
apache/shardingsphere-proxy-test:latest
+ - uses: actions/upload-artifact@v4
+ with:
+ name: e2e-image
+ path: /tmp/${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-proxy-test.tar
+ retention-days: 10
+
+ e2e-sql:
+ name: E2E - SQL
+ needs: [ build-e2e-image, global-environment ]
+ if: ${{ needs.global-environment.outputs.GLOBAL_JOB_ENABLED == 'true' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ strategy:
+ max-parallel: 20
+ fail-fast: false
+ matrix:
+ adapter: ${{ fromJson(github.event.inputs.adapter) }}
+ mode: ${{ fromJson(github.event.inputs.mode) }}
+ database: ${{ fromJson(github.event.inputs.database) }}
+ # Fix me #25051
+ #scenario: [ dbtbl_with_readwrite_splitting,
dbtbl_with_readwrite_splitting_and_encrypt, sharding_and_encrypt,
encrypt_and_readwrite_splitting, encrypt_shadow,
readwrite_splitting_and_shadow, sharding_and_shadow, sharding_encrypt_shadow,
mask_encrypt, mask_sharding, mask_encrypt_sharding ]
+ scenario: ${{ fromJson(github.event.inputs.scenario) }}
+ additional-options: ${{
fromJson(github.event.inputs.additional-options) }}
+ include:
+ - adapter: proxy
+ database: MySQL
+ scenario: passthrough
+ additional-options: '-Dmysql-connector-java.version=8.3.0'
+ exclude:
+ - adapter: jdbc
+ scenario: passthrough
+ - adapter: jdbc
+ mode: Cluster
+ - adapter: proxy
+ mode: Standalone
+ scenario: empty_rules
+ - adapter: proxy
+ mode: Standalone
+ scenario: distsql_rdl
+ - adapter: proxy
+ mode: Standalone
+ scenario: passthrough
+ - adapter: proxy
+ scenario: empty_storage_units
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX
}}-maven-third-party-e2e-cache-${{ github.sha }}
+ - uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: 21
+ - name: Download E2E Image
+ if: matrix.adapter == 'proxy'
+ uses: actions/download-artifact@v4
+ with:
+ name: e2e-image
+ path: /tmp/
+ - name: Load E2E Image
+ if: matrix.adapter == 'proxy'
+ run: docker load -i /tmp/${{
needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-proxy-test.tar
+ - name: Run E2E Test
+ run: ./mvnw -nsu -B install -f test/e2e/sql/pom.xml
-Dspotless.apply.skip=true -Dit.cluster.env.type=DOCKER
-Dit.cluster.adapters=${{ matrix.adapter }} -Dit.run.modes=${{ matrix.mode }}
-Dit.run.additional.cases=false -Dit.scenarios=${{ matrix.scenario }}
-Dit.cluster.databases=${{ matrix.database }} ${{ matrix.additional-options }}
