This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new b3aa2e89489 Fix Python PreCommit dependency compatibility test not run
or redundant setup (#25115)
b3aa2e89489 is described below
commit b3aa2e89489898f8c760294ba4dba2310ac53e70
Author: Yi Hu <[email protected]>
AuthorDate: Mon Jan 23 12:12:40 2023 -0500
Fix Python PreCommit dependency compatibility test not run or redundant
setup (#25115)
* Fix Python PreCommit dependency compatibility test not run or redundant
setup
* Move pyarrow and pandas matrix unit test to preCommitPyCoverage
* Fix Dataframe compatibility test not run
* Update readme
* Also move dataframe comp tests to preCommitPyCoverage
---
.test-infra/jenkins/README.md | 1 +
sdks/python/test-suites/tox/py38/build.gradle | 81 +++++++++++++--------------
2 files changed, 40 insertions(+), 42 deletions(-)
diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md
index 0b7b20f5433..a99d4b0c9bd 100644
--- a/.test-infra/jenkins/README.md
+++ b/.test-infra/jenkins/README.md
@@ -47,6 +47,7 @@ Beam Jenkins overview page:
[link](https://ci-beam.apache.org/)
| beam_PreCommit_Python_Examples |
[commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Commit/),
[cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Cron/),
[phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Phrase/)
| `Run Python_Examples PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Cron)
|
| beam_PreCommit_Python_Runners |
[commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Commit/),
[cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Cron/),
[phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Phrase/)
| `Run Python_Runners PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Cron)
|
| beam_PreCommit_Python_Transforms |
[commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Commit/),
[cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Cron/),
[phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Phrase/)
| `Run Python_Transforms PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Cron)
|
+| beam_PreCommit_Python_Coverage |
[commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Commit/),
[cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Cron/),
[phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Phrase/)
| `Run Python_Coverage PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Cron)
|
| beam_PreCommit_PythonDocker |
[commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Commit/),
[cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/),
[phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Phrase/) |
`Run PythonDocker PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/)
|
| beam_PreCommit_PythonDocs|
[commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Commit/),
[cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/),
[phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Phrase/) |
`Run PythonDocs PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/)
|
| beam_PreCommit_Python_PVR_Flink |
[commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Commit/),
[cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/),
[phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Phrase/)
| `Run Python_PVR_Flink PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron)
|
diff --git a/sdks/python/test-suites/tox/py38/build.gradle
b/sdks/python/test-suites/tox/py38/build.gradle
index 072beb172cf..e90b306f698 100644
--- a/sdks/python/test-suites/tox/py38/build.gradle
+++ b/sdks/python/test-suites/tox/py38/build.gradle
@@ -36,82 +36,83 @@ testPy38Cython.mustRunAfter testPython38,
testPy38CloudCoverage
// TODO(BEAM-12000): Move tasks that aren't specific to 3.8 to Py 3.9.
def posargs = project.findProperty("posargs") ?: ""
-def runDataframesTests = (posargs == "" || posargs.contains("dataframes"))
-logger.info('Running dataframes tests: ' + runDataframesTests)
+
+// PyCoverage Precommit runs test suites that evaluate test coverage and
compatibility of
+// particular dependencies. It is exercised on a single Python version.
+project.tasks.register("preCommitPyCoverage") {
+ dependsOn = ["testPy38CloudCoverage"]
+}
// Create a test task for each major version of pyarrow
toxTask "testPy38pyarrow-0", "py38-pyarrow-0", "${posargs}"
test.dependsOn "testPy38pyarrow-0"
-preCommitPy38.dependsOn "testPy38pyarrow-0"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-0"
toxTask "testPy38pyarrow-1", "py38-pyarrow-1", "${posargs}"
test.dependsOn "testPy38pyarrow-1"
-preCommitPy38.dependsOn "testPy38pyarrow-1"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-1"
toxTask "testPy38pyarrow-2", "py38-pyarrow-2", "${posargs}"
test.dependsOn "testPy38pyarrow-2"
-preCommitPy38.dependsOn "testPy38pyarrow-2"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-2"
toxTask "testPy38pyarrow-3", "py38-pyarrow-3", "${posargs}"
test.dependsOn "testPy38pyarrow-3"
-preCommitPy38.dependsOn "testPy38pyarrow-3"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-3"
toxTask "testPy38pyarrow-4", "py38-pyarrow-4", "${posargs}"
test.dependsOn "testPy38pyarrow-4"
-preCommitPy38.dependsOn "testPy38pyarrow-4"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-4"
toxTask "testPy38pyarrow-5", "py38-pyarrow-5", "${posargs}"
test.dependsOn "testPy38pyarrow-5"
-preCommitPy38.dependsOn "testPy38pyarrow-5"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-5"
toxTask "testPy38pyarrow-6", "py38-pyarrow-6", "${posargs}"
test.dependsOn "testPy38pyarrow-6"
-preCommitPy38.dependsOn "testPy38pyarrow-6"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-6"
toxTask "testPy38pyarrow-7", "py38-pyarrow-7", "${posargs}"
test.dependsOn "testPy38pyarrow-7"
-preCommitPy38.dependsOn "testPy38pyarrow-7"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-7"
toxTask "testPy38pyarrow-8", "py38-pyarrow-8", "${posargs}"
test.dependsOn "testPy38pyarrow-8"
-preCommitPy38.dependsOn "testPy38pyarrow-8"
+preCommitPyCoverage.dependsOn "testPy38pyarrow-8"
toxTask "testPy38pyarrow-9", "py38-pyarrow-9", "${posargs}"
test.dependsOn "testPy38pyarrow-9"
-preCommitPy38.dependsOn "testPy38pyarrow-9"
-
-// Dataframes tests won't get auto-filtered by path since they explicitly only
run on the dataframes directory in tox.ini
-if (runDataframesTests) {
- // Create a test task for each minor version of pandas
- toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}"
- test.dependsOn "testPy38pandas-11"
- preCommitPy38.dependsOn "testPy38pandas-11"
-
- toxTask "testPy38pandas-12", "py38-pandas-12", "${posargs}"
- test.dependsOn "testPy38pandas-12"
- preCommitPy38.dependsOn "testPy38pandas-12"
-
- toxTask "testPy38pandas-13", "py38-pandas-13", "${posargs}"
- test.dependsOn "testPy38pandas-13"
- preCommitPy38.dependsOn "testPy38pandas-13"
-
- toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}"
- test.dependsOn "testPy38pandas-14"
- preCommitPy38.dependsOn "testPy38pandas-14"
-
- toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}"
- test.dependsOn "testPy38pandas-15"
- preCommitPy38.dependsOn "testPy38pandas-15"
-}
+preCommitPyCoverage.dependsOn "testPy38pyarrow-9"
+
+// Create a test task for each minor version of pandas
+toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}"
+test.dependsOn "testPy38pandas-11"
+preCommitPyCoverage.dependsOn "testPy38pandas-11"
+
+toxTask "testPy38pandas-12", "py38-pandas-12", "${posargs}"
+test.dependsOn "testPy38pandas-12"
+preCommitPyCoverage.dependsOn "testPy38pandas-12"
+
+toxTask "testPy38pandas-13", "py38-pandas-13", "${posargs}"
+test.dependsOn "testPy38pandas-13"
+preCommitPyCoverage.dependsOn "testPy38pandas-13"
+
+toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}"
+test.dependsOn "testPy38pandas-14"
+preCommitPyCoverage.dependsOn "testPy38pandas-14"
+
+toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}"
+test.dependsOn "testPy38pandas-15"
+preCommitPyCoverage.dependsOn "testPy38pandas-15"
// Create a test task for each minor version of pytorch
toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}"
test.dependsOn "testPy38pytorch-19"
-preCommitPy38.dependsOn "testPy38pytorch-19"
+preCommitPyCoverage.dependsOn "testPy38pytorch-19"
toxTask "testPy38pytorch-110", "py38-pytorch-110", "${posargs}"
test.dependsOn "testPy38pytorch-110"
-preCommitPy38.dependsOn "testPy38pytorch-110"
+preCommitPyCoverage.dependsOn "testPy38pytorch-110"
toxTask "whitespacelint", "whitespacelint", "${posargs}"
@@ -126,10 +127,6 @@ task archiveFilesToLint(type: Zip) {
}
}
-project.tasks.register("preCommitPyCoverage") {
- dependsOn = ["testPy38CloudCoverage"]
-}
-
task unpackFilesToLint(type: Copy) {
from zipTree("$buildDir/dist/files-to-whitespacelint.zip")
into "$buildDir/files-to-whitespacelint"
