This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7ca1a5ddffd Move dependency tests from PreCommit Coverage to
PostCommit (#30704)
7ca1a5ddffd is described below
commit 7ca1a5ddffdc777a6831d2d4c1d3b6dcd9905219
Author: Yi Hu <[email protected]>
AuthorDate: Fri Mar 22 14:41:11 2024 -0400
Move dependency tests from PreCommit Coverage to PostCommit (#30704)
* Move dependency tests from PreCommit Coverage to PostCommit
* Move at least one version to precommit
---
.github/workflows/README.md | 1 +
.../beam_PostCommit_Python_Dependency.yml | 99 ++++++++++++++++++++++
sdks/python/test-suites/tox/common.gradle | 4 -
sdks/python/test-suites/tox/py38/build.gradle | 54 +++++++-----
4 files changed, 132 insertions(+), 26 deletions(-)
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index 74c9ee90868..2a36fea9ab0 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -341,6 +341,7 @@ PostCommit Jobs run in a schedule against master branch and
generally do not get
| [ PostCommit PortableJar Spark
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml)
| N/A |`beam_PostCommit_PortableJar_Spark.json`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml?query=event%3Aschedule)
|
| [ PostCommit Python
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml)
| ['3.8','3.9','3.10','3.11'] |`beam_PostCommit_Python.json`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml?query=event%3Aschedule)
|
| [ PostCommit Python
Arm](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml)
| ['3.8','3.9','3.10','3.11'] |`beam_PostCommit_Python_Arm.json`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml?query=event%3Aschedule)
|
+| [ PostCommit Python Dependency
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Dependency.yml)
| N/A |`beam_PostCommit_Python_Dependency.json`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Dependency.yml?query=event%3Aschedule)
|
| [ PostCommit Python Examples Dataflow
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml)
| N/A |`beam_PostCommit_Python_Examples_Dataflow.json`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml?query=event%3Asch
[...]
| [ PostCommit Python Examples Direct
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml)
| ['3.8','3.9','3.10','3.11'] |`beam_PostCommit_Python_Examples_Direct.json`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml?query
[...]
| [ PostCommit Python Examples Flink
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml)
| ['3.8','3.11'] |`beam_PostCommit_Python_Examples_Flink.json`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml?query=event%3Aschedule)
|
diff --git a/.github/workflows/beam_PostCommit_Python_Dependency.yml
b/.github/workflows/beam_PostCommit_Python_Dependency.yml
new file mode 100644
index 00000000000..5707d06a6a6
--- /dev/null
+++ b/.github/workflows/beam_PostCommit_Python_Dependency.yml
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: PostCommit Python Dependency
+
+on:
+ schedule:
+ - cron: '0 5/6 * * *'
+ pull_request_target:
+ paths: ['release/trigger_all_tests.json',
'.github/trigger_files/beam_PostCommit_Python_Dependency.json']
+ workflow_dispatch:
+
+#Setting explicit permissions for the action to avoid the default permissions
which are `write-all` in case of pull_request_target event
+permissions:
+ actions: write
+ pull-requests: write
+ checks: write
+ contents: read
+ deployments: read
+ id-token: none
+ issues: write
+ discussions: read
+ packages: read
+ pages: read
+ repository-projects: read
+ security-events: read
+ statuses: read
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+ group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha
|| github.head_ref || github.ref }}-${{ github.event.schedule ||
github.event.comment.id || github.event.sender.login }}'
+ cancel-in-progress: true
+
+env:
+ GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+ GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+ HF_INFERENCE_TOKEN: ${{ secrets.HF_INFERENCE_TOKEN }}
+
+
+jobs:
+ beam_PostCommit_Python_Dependency:
+ name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+ runs-on: [self-hosted, ubuntu-20.04, main]
+ strategy:
+ fail-fast: false
+ matrix:
+ job_name: [beam_PostCommit_Python_Dependency]
+ job_phrase: [Run Python PostCommit Dependency]
+ timeout-minutes: 120
+ if: |
+ github.event_name == 'workflow_dispatch' ||
+ github.event_name == 'pull_request_target' ||
+ (github.event_name == 'schedule' && github.repository == 'apache/beam')
||
+ github.event.comment.body == 'Run Python PostCommit Dependency'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup repository
+ uses: ./.github/actions/setup-action
+ with:
+ comment_phrase: ${{ matrix.job_phrase }}
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+ - name: Setup environment
+ uses: ./.github/actions/setup-environment-action
+ with:
+ java-version: 8
+ python-version: 3.8
+ - name: Run postCommitPyDep
+ uses: ./.github/actions/gradle-command-self-hosted-action
+ with:
+ gradle-command: :sdks:python:test-suites:tox:py38:postCommitPyDep
+ arguments: |
+ -PuseWheelDistribution
+ - name: Archive Python Test Results
+ uses: actions/upload-artifact@v4
+ if: failure()
+ with:
+ name: Python Test Results
+ path: '**/pytest*.xml'
+ - name: Publish Python Test Results
+ uses: EnricoMi/publish-unit-test-result-action@v2
+ if: always()
+ with:
+ commit: '${{ env.prsha || env.GITHUB_SHA }}'
+ comment_mode: ${{ github.event_name == 'issue_comment' && 'always'
|| 'off' }}
+ files: '**/pytest*.xml'
diff --git a/sdks/python/test-suites/tox/common.gradle
b/sdks/python/test-suites/tox/common.gradle
index aec92a7a8cb..5a8055b36e5 100644
--- a/sdks/python/test-suites/tox/common.gradle
+++ b/sdks/python/test-suites/tox/common.gradle
@@ -29,10 +29,6 @@ test.dependsOn "testPy${pythonVersionSuffix}Cloud"
// toxTask "testPy${pythonVersionSuffix}Dask",
"py${pythonVersionSuffix}-dask", "${posargs}"
// test.dependsOn "testPy${pythonVersionSuffix}Dask"
-
-toxTask "testPy38CloudCoverage", "py38-cloudcoverage", "${posargs}"
-test.dependsOn "testPy38CloudCoverage"
-
project.tasks.register("preCommitPy${pythonVersionSuffix}") {
// Since codecoverage reports will always be generated for py38,
// all tests will be exercised.
diff --git a/sdks/python/test-suites/tox/py38/build.gradle
b/sdks/python/test-suites/tox/py38/build.gradle
index 60245e84c73..46388532c1e 100644
--- a/sdks/python/test-suites/tox/py38/build.gradle
+++ b/sdks/python/test-suites/tox/py38/build.gradle
@@ -44,76 +44,84 @@ lint.dependsOn mypyPy38
apply from: "../common.gradle"
-
-// PyCoverage Precommit runs test suites that evaluate test coverage and
compatibility of
-// particular dependencies. It is exercised on a single Python version.
+toxTask "testPy38CloudCoverage", "py38-cloudcoverage", "${posargs}"
+test.dependsOn "testPy38CloudCoverage"
project.tasks.register("preCommitPyCoverage") {
dependsOn = ["testPy38CloudCoverage"]
}
+// Dep Postcommit runs test suites that evaluate compatibility of particular
+// dependencies. It is exercised on a single Python version.
+//
+// Should still leave at least one version in PreCommit unless the marked tests
+// are also exercised by existing PreCommit
+// e.g. pyarrow and pandas also run on PreCommit Dataframe and Coverage
+project.tasks.register("postCommitPyDep") {}
+
// Create a test task for each supported major version of pyarrow
toxTask "testPy38pyarrow-3", "py38-pyarrow-3", "${posargs}"
test.dependsOn "testPy38pyarrow-3"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-3"
+postCommitPyDep.dependsOn "testPy38pyarrow-3"
toxTask "testPy38pyarrow-4", "py38-pyarrow-4", "${posargs}"
test.dependsOn "testPy38pyarrow-4"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-4"
+postCommitPyDep.dependsOn "testPy38pyarrow-4"
toxTask "testPy38pyarrow-5", "py38-pyarrow-5", "${posargs}"
test.dependsOn "testPy38pyarrow-5"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-5"
+postCommitPyDep.dependsOn "testPy38pyarrow-5"
toxTask "testPy38pyarrow-6", "py38-pyarrow-6", "${posargs}"
test.dependsOn "testPy38pyarrow-6"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-6"
+postCommitPyDep.dependsOn "testPy38pyarrow-6"
toxTask "testPy38pyarrow-7", "py38-pyarrow-7", "${posargs}"
test.dependsOn "testPy38pyarrow-7"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-7"
+postCommitPyDep.dependsOn "testPy38pyarrow-7"
toxTask "testPy38pyarrow-8", "py38-pyarrow-8", "${posargs}"
test.dependsOn "testPy38pyarrow-8"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-8"
+postCommitPyDep.dependsOn "testPy38pyarrow-8"
toxTask "testPy38pyarrow-9", "py38-pyarrow-9", "${posargs}"
test.dependsOn "testPy38pyarrow-9"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-9"
+postCommitPyDep.dependsOn "testPy38pyarrow-9"
// Create a test task for each minor version of pandas
toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}"
test.dependsOn "testPy38pandas-14"
-preCommitPyCoverage.dependsOn "testPy38pandas-14"
+postCommitPyDep.dependsOn "testPy38pandas-14"
toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}"
test.dependsOn "testPy38pandas-15"
-preCommitPyCoverage.dependsOn "testPy38pandas-15"
+postCommitPyDep.dependsOn "testPy38pandas-15"
toxTask "testPy38pandas-20", "py38-pandas-20", "${posargs}"
test.dependsOn "testPy38pandas-20"
-preCommitPyCoverage.dependsOn "testPy38pandas-20"
+postCommitPyDep.dependsOn "testPy38pandas-20"
// Create a test task for each minor version of pytorch
toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}"
test.dependsOn "testPy38pytorch-19"
-preCommitPyCoverage.dependsOn "testPy38pytorch-19"
+postCommitPyDep.dependsOn "testPy38pytorch-19"
toxTask "testPy38pytorch-110", "py38-pytorch-110", "${posargs}"
test.dependsOn "testPy38pytorch-110"
-preCommitPyCoverage.dependsOn "testPy38pytorch-110"
+postCommitPyDep.dependsOn "testPy38pytorch-110"
toxTask "testPy38pytorch-111", "py38-pytorch-111", "${posargs}"
test.dependsOn "testPy38pytorch-111"
-preCommitPyCoverage.dependsOn "testPy38pytorch-111"
+postCommitPyDep.dependsOn "testPy38pytorch-111"
toxTask "testPy38pytorch-112", "py38-pytorch-112", "${posargs}"
test.dependsOn "testPy38pytorch-112"
-preCommitPyCoverage.dependsOn "testPy38pytorch-112"
+postCommitPyDep.dependsOn "testPy38pytorch-112"
toxTask "testPy38pytorch-113", "py38-pytorch-113", "${posargs}"
test.dependsOn "testPy38pytorch-113"
-preCommitPyCoverage.dependsOn "testPy38pytorch-113"
+postCommitPyDep.dependsOn "testPy38pytorch-113"
+// run on precommit
toxTask "testPy38pytorch-200", "py38-pytorch-200", "${posargs}"
test.dependsOn "testPy38pytorch-200"
preCommitPyCoverage.dependsOn "testPy38pytorch-200"
@@ -126,7 +134,7 @@ preCommitPyCoverage.dependsOn "testPy38tft-113"
// Create a test task for each minor version of onnx
// toxTask "testPy38onnx-113", "py38-onnx-113", "${posargs}"
// test.dependsOn "testPy38onnx-113"
-// preCommitPyCoverage.dependsOn "testPy38onnx-113"
+// postCommitPyDep.dependsOn "testPy38onnx-113"
// Create a test task for each minor version of tensorflow
toxTask "testPy38tensorflow-212", "py38-tensorflow-212", "${posargs}"
test.dependsOn "testPy38tensorflow-212"
@@ -135,12 +143,13 @@ preCommitPyCoverage.dependsOn "testPy38tensorflow-212"
// Create a test task for each minor version of transformers
toxTask "testPy38transformers-428", "py38-transformers-428", "${posargs}"
test.dependsOn "testPy38transformers-428"
-preCommitPyCoverage.dependsOn "testPy38transformers-428"
+postCommitPyDep.dependsOn "testPy38transformers-428"
toxTask "testPy38transformers-429", "py38-transformers-429", "${posargs}"
test.dependsOn "testPy38transformers-429"
-preCommitPyCoverage.dependsOn "testPy38transformers-429"
+postCommitPyDep.dependsOn "testPy38transformers-429"
+// run on precommit
toxTask "testPy38transformers-430", "py38-transformers-430", "${posargs}"
test.dependsOn "testPy38transformers-430"
preCommitPyCoverage.dependsOn "testPy38transformers-430"
@@ -153,8 +162,9 @@ preCommitPyCoverage.dependsOn
"testPy38embeddingsMLTransform"
// mutliple versions so keeping this suite separate.
toxTask "testPy38TensorflowHubEmbeddings-014", "py38-TFHubEmbeddings-014",
"${posargs}"
test.dependsOn "testPy38TensorflowHubEmbeddings-014"
-preCommitPyCoverage.dependsOn "testPy38TensorflowHubEmbeddings-014"
+postCommitPyDep.dependsOn "testPy38TensorflowHubEmbeddings-014"
+// run on precommit
toxTask "testPy38TensorflowHubEmbeddings-015", "py38-TFHubEmbeddings-015",
"${posargs}"
test.dependsOn "testPy38TensorflowHubEmbeddings-015"
preCommitPyCoverage.dependsOn "testPy38TensorflowHubEmbeddings-015"
