This is an automated email from the ASF dual-hosted git repository.
zhaojinchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new b3627224b19 Add : add e2e operation workflow (#26910) (#26923)
b3627224b19 is described below
commit b3627224b195fadff94c00aa0fff56170ed731a9
Author: 孙念君 Nianjun Sun <[email protected]>
AuthorDate: Fri Jul 14 14:41:22 2023 +0800
Add : add e2e operation workflow (#26910) (#26923)
* Add : add e2e operation workflow (#26910)
* Add : add filter files for paths-fileter action (#26910)
* Refactor : add default value for showprocesslist it.scenarios
* Refactor : add default value for showprocesslist run.modes
* Refactor : add default value for showprocesslist
* Refactor : refactor the key names for showprocesslist
* Refactor : refactor the workflow
* Refactor : remove the id content
* Refactor : refactor the function name
* Refactor : refactor the function
* Refactor : add id properties
* Refactor : refactor the outputs position
* Refactor : try to remve the name property in steps
* Refactor : extract the shell script into sh file
* Refactor : extract the shell script into function
* Refactor : extract the shell script into function
* Refactor : extract the shell script into function
* Refactor : refactor the shell to matrix
* Refactor : refactor the matrix name
* Refactor : refactor the matrix name
* Refactor : refactor the matrix name
* Fix : fix the incorrect usage of image version
* Refactor : refactor the name of test matrix
* Refactor : refactor the name of test matrix step
* Format : format the yml file
* Refactor : refactor the script
* Refactor : update the filter config file (#26910)
---
.github/workflows/e2e-operation.yml | 113 +++++++++++++++++++++
.github/workflows/resources/filter/filters.yml | 62 +++++++++++
.../env/ShowProcessListEnvironment.java | 8 +-
.../src/test/resources/env/it-env.properties | 18 ++--
4 files changed, 188 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/e2e-operation.yml
b/.github/workflows/e2e-operation.yml
new file mode 100644
index 00000000000..9a9f84ef3cf
--- /dev/null
+++ b/.github/workflows/e2e-operation.yml
@@ -0,0 +1,113 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: E2E - Operation
+
+on:
+ push:
+ branches: [ master, dev ]
+ pull_request:
+ schedule:
+ - cron: '0 22 */1 * *' # once a day. UTC time
+ workflow_dispatch:
+
+concurrency:
+ group: e2e-pipeline-${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3 -Dmaven.javadoc.skip=true
-Dcheckstyle.skip=true -Dspotless.apply.skip=true -Djacoco.skip=true
-Dfailsafe.skipAfterFailureCount=1 -Dio.netty.leakDetectionLevel=advanced
+ REPOSITORY_NAME: shardingsphere
+
+jobs:
+ detect-changed-files:
+ name: Detect Changed Files
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - uses: actions/checkout@v3
+ - uses: dorny/paths-filter@v2
+ id: filter
+ with:
+ token: ${{ github.token }}
+ filters: .github/workflows/resources/filter/filters.yml
+ outputs:
+ changed_operations: ${{ steps.filter.outputs.changes }}
+
+ e2e-operation-job:
+ needs: detect-changed-files
+ name: E2E - ${{ matrix.operation }} on ${{ matrix.image.version }}
+ runs-on: ubuntu-latest
+ strategy:
+ max-parallel: 5
+ fail-fast: false
+ matrix:
+ operation: [ transaction, pipeline, showprocesslist ]
+ image: [ { type: "it.docker.mysql.version", version: "mysql:5.7" }, {
type: "it.docker.postgresql.version", version: "postgres:12-alpine" }, { type:
"it.docker.opengauss.version", version: "enmotech/opengauss:2.1.0" } ]
+ exclude:
+ - operation: showprocesslist
+ image: { type: "it.docker.postgresql.version", version:
"postgres:12-alpine" }
+ - operation: showprocesslist
+ image: { type: "it.docker.opengauss.version", version:
"enmotech/opengauss:2.1.0" }
+ timeout-minutes: 40
+ steps:
+ - env:
+ changed_operations: ${{
needs.detect-changed-files.outputs.changed_operations }}
+ current_operation: ${{ matrix.operation }}
+ run: |
+ echo "skip_current_step=false" >> $GITHUB_ENV
+ if [[ ${changed_operations} == '["ignore"]' ]]; then
+ echo "${current_operation} is ignore by ignore filter"
+ echo "skip_current_step=true" >> $GITHUB_ENV
+ fi
+
+ if [[ ${changed_operations} == *""$current_operation""* ]]; then
+ echo "${current_operation} is detected by changed filter"
+ else
+ echo "${current_operation} is ignore by filter"
+ echo "skip_current_step=true" >> $GITHUB_ENV
+ fi
+ - name: Checkout Project
+ if: (env.skip_current_step == 'false')
+ uses: actions/checkout@v3
+ - name: Retrieve Maven Caches
+ if: (env.skip_current_step == 'false')
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2/repository
+ key: ${{ env.REPOSITORY_NAME }}-maven-third-party-e2e-cache-${{
github.sha }}
+ restore-keys: |
+ ${{ env.REPOSITORY_NAME }}-maven-third-party-e2e-cache-
+ ${{ env.REPOSITORY_NAME }}-maven-third-party-
+ - name: Setup JDK 11
+ if: (env.skip_current_step == 'false')
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: 11
+ - name: Build ${{ matrix.operation }} E2E Image
+ if: (env.skip_current_step == 'false')
+ run: ./mvnw -B clean install -am -pl test/e2e/operation/${{
matrix.operation }} -Pit.env.docker -DskipTests
+ - name: Setup JDK 8 for Test
+ if: (env.skip_current_step == 'false')
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: 8
+ - name: Run ${{ matrix.operation }} on ${{ matrix.image.version }}
+ if: (env.skip_current_step == 'false')
+ run: ./mvnw -nsu -B install -f test/e2e/operation/${{ matrix.operation
}}/pom.xml -D${{ matrix.operation }}.it.env.type=docker -D${{ matrix.operation
}}.${{ matrix.image.type }}=${{ matrix.image.version }}
diff --git a/.github/workflows/resources/filter/filters.yml
b/.github/workflows/resources/filter/filters.yml
new file mode 100644
index 00000000000..ebe98bddcd8
--- /dev/null
+++ b/.github/workflows/resources/filter/filters.yml
@@ -0,0 +1,62 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ignore:
+ - '**/*.md'
+
+pipeline:
+ - '.github/workflows/e2e-operation.yml'
+ - 'infra/**/src/main/**'
+ - 'mode/**/src/main/**'
+ - 'features/encrypt/src/main/**'
+ - 'features/features/readwrite-splitting/src/main/**'
+ - 'features/sharding/src/main/**'
+ - 'proxy/**/src/main/**'
+ - 'jdbc/core/src/main/**'
+ - '**/*-distsql*/**/src/main/**'
+ - 'kernel/data-pipeline/**/src/main/**'
+ - 'kernel/data-pipeline/**/pom.xml'
+ - '**/src/main/**/data/pipeline/**'
+ - '**/src/main/resources/META-INF/services/*.data.pipeline.*'
+ - 'test/pom.xml'
+ - 'test/e2e/fixture/**'
+ - 'test/e2e/env/**'
+ - 'test/e2e/operation/pipeline/**'
+
+showprocesslist:
+ - '.github/workflows/e2e-operation.yml'
+ - 'infra/common/src/main/**'
+ - 'mode/**/src/main/**'
+ - 'proxy/**/src/main/**'
+ - 'jdbc/core/src/main/**'
+ - 'test/pom.xml'
+ - 'test/e2e/fixture/**'
+ - 'test/e2e/env/**'
+ - 'test/e2e/operation/showprocesslist/**'
+
+transaction:
+ - '.github/workflows/e2e-operation.yml'
+ - 'infra/common/src/main/**'
+ - 'mode/**/src/main/**'
+ - 'proxy/**/src/main/**'
+ - 'jdbc/core/src/main/**'
+ - 'kernel/transaction/**/src/main/**'
+ - 'kernel/transaction/**/pom.xml'
+ - 'test/pom.xml'
+ - 'test/e2e/fixture/**'
+ - 'test/e2e/env/**'
+ - 'test/e2e/operation/transaction/**'
diff --git
a/test/e2e/operation/showprocesslist/src/test/java/org/apache/shardingsphere/test/e2e/showprocesslist/env/ShowProcessListEnvironment.java
b/test/e2e/operation/showprocesslist/src/test/java/org/apache/shardingsphere/test/e2e/showprocesslist/env/ShowProcessListEnvironment.java
index 73dfe5430ca..e490b245924 100644
---
a/test/e2e/operation/showprocesslist/src/test/java/org/apache/shardingsphere/test/e2e/showprocesslist/env/ShowProcessListEnvironment.java
+++
b/test/e2e/operation/showprocesslist/src/test/java/org/apache/shardingsphere/test/e2e/showprocesslist/env/ShowProcessListEnvironment.java
@@ -45,10 +45,10 @@ public final class ShowProcessListEnvironment {
private ShowProcessListEnvironment() {
props = loadProperties();
- runModes =
Splitter.on(",").trimResults().splitToList(props.getProperty("it.run.modes",
"Standalone, Cluster"));
- itEnvType =
ShowProcessListEnvTypeEnum.valueOf(props.getProperty("it.env.type",
ShowProcessListEnvTypeEnum.NONE.name()).toUpperCase());
+ runModes =
Splitter.on(",").trimResults().splitToList(props.getProperty("showprocesslist.it.run.modes",
"Standalone,Cluster"));
+ itEnvType =
ShowProcessListEnvTypeEnum.valueOf(props.getProperty("showprocesslist.it.env.type",
ShowProcessListEnvTypeEnum.NONE.name()).toUpperCase());
scenarios = getScenarios(props);
- governanceCenters =
Splitter.on(",").trimResults().splitToList(props.getProperty("it.governance.centers"));
+ governanceCenters =
Splitter.on(",").trimResults().splitToList(props.getProperty("showprocesslist.it.governance.centers"));
}
/**
@@ -73,7 +73,7 @@ public final class ShowProcessListEnvironment {
}
private Collection<String> getScenarios(final Properties props) {
- Collection<String> result =
Splitter.on(",").trimResults().splitToList(props.getProperty("it.scenarios"));
+ Collection<String> result =
Splitter.on(",").trimResults().splitToList(props.getProperty("showprocesslist.it.scenarios",
"cluster_jdbc_proxy"));
for (String each : result) {
new ScenarioCommonPath(each).checkFolderExist();
}
diff --git
a/test/e2e/operation/showprocesslist/src/test/resources/env/it-env.properties
b/test/e2e/operation/showprocesslist/src/test/resources/env/it-env.properties
index bd245c0ee1e..bfcc5c6b49c 100644
---
a/test/e2e/operation/showprocesslist/src/test/resources/env/it-env.properties
+++
b/test/e2e/operation/showprocesslist/src/test/resources/env/it-env.properties
@@ -1,4 +1,4 @@
-#
+\#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -15,11 +15,11 @@
# limitations under the License.
#
-# it.env.type=NONE,DOCKER
-it.env.type=NONE
-# it.scenarios=cluster_jdbc_proxy
-it.scenarios=cluster_jdbc_proxy
-#it.run.modes=Standalone,Cluster
-it.run.modes=Standalone,Cluster
-#it.governance.center=Zookeeper
-it.governance.centers=Zookeeper
+# showprocesslist.it.env.type=NONE,DOCKER
+showprocesslist.it.env.type=NONE
+# showprocesslist.it.scenarios=cluster_jdbc_proxy
+showprocesslist.it.scenarios=cluster_jdbc_proxy
+# showprocesslist.it.run.modes=Standalone,Cluster
+showprocesslist.it.run.modes=Standalone,Cluster
+# showprocesslist.it.governance.center=Zookeeper
+showprocesslist.it.governance.centers=Zookeeper
