This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a9516ba18a7 Add arm tests to postcommit suite (#28644)
a9516ba18a7 is described below
commit a9516ba18a7569524fd4f8092caeb12ff4c02bf6
Author: Danny McCormick <[email protected]>
AuthorDate: Thu Sep 28 16:50:24 2023 -0400
Add arm tests to postcommit suite (#28644)
* Add arm tests to postcommit suite
* Add separate arm suite
* Temporarily add pull_request target
* Remove pull request trigger
* Temporarily just run on workflow_dispatch
* Add to readme
---
.github/workflows/README.md | 3 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 103 +++++++++++++++++++++++
sdks/python/scripts/run_integration_test.sh | 10 +++
sdks/python/test-suites/dataflow/common.gradle | 20 +++++
4 files changed, 135 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index f360bc15ae2..03479e5c4c3 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -246,6 +246,7 @@ Please note that jobs with matrix need to have matrix
element in the comment. Ex
| [ PostCommit PortableJar Flink
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml)
| N/A |`Run PortableJar_Flink PostCommit`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml)
|
| [ PostCommit PortableJar Spark
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml)
| N/A |`Run PortableJar_Spark PostCommit`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml)
|
| [ PostCommit Python
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml)
| ['3.8','3.9','3.10','3.11'] |`Run Python PostCommit (matrix_element)`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml)
|
+| [ PostCommit Python
Arm](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml)
| ['3.8','3.9','3.10','3.11'] |`Run Python PostCommit Arm (matrix_element)`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml)
|
| [ PostCommit Python Examples Dataflow
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml)
| N/A |`Run Python Examples_Dataflow`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml)
|
| [ PostCommit Python Examples Direct
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml)
| ['3.8','3.9','3.10','3.11'] |`Run Python Examples_Direct (matrix_element)`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml)
|
| [ PostCommit Python Examples Flink
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml)
| ['3.8','3.11'] |`Run Python Examples_Flink (matrix_element)`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml)
|
@@ -346,4 +347,4 @@ Please note that jobs with matrix need to have matrix
element in the comment. Ex
| [ PreCommit Portable Python
](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Portable_Python.yml)
| ['3.8','3.11'] | `Run Portable_Python PreCommit` |
[](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Portable_Python.yml)
|
| [ Cancel Stale Dataflow Jobs
](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml)
| N/A | `Run Cancel Stale Dataflow Jobs` |
[](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml)
|
| [ Clean Up GCP Resources
](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml)
| N/A | `Run Clean GCP Resources` |
[](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml)
|
-| [ Clean Up Prebuilt SDK Images
](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml)
| N/A | `Run Clean Prebuilt Images` |
[](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml)
|
\ No newline at end of file
+| [ Clean Up Prebuilt SDK Images
](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml)
| N/A | `Run Clean Prebuilt Images` |
[](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml)
|
diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml
b/.github/workflows/beam_PostCommit_Python_Arm.yml
new file mode 100644
index 00000000000..74eb8e8290d
--- /dev/null
+++ b/.github/workflows/beam_PostCommit_Python_Arm.yml
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PostCommit Python Arm
+
+on:
+ # issue_comment:
+ # types: [created]
+ # schedule:
+ # - cron: '0 */6 * * *'
+ workflow_dispatch:
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+ group: '${{ github.workflow }} @ ${{ github.event.issue.number ||
github.event.pull_request.head.label || github.sha || github.head_ref ||
github.ref }}-${{ github.event.schedule || github.event.comment.body ||
github.event.sender.login}}'
+ cancel-in-progress: true
+
+#Setting explicit permissions for the action to avoid the default permissions
which are `write-all` in case of pull_request_target event
+permissions:
+ actions: write
+ pull-requests: read
+ checks: read
+ contents: read
+ deployments: read
+ id-token: none
+ issues: read
+ discussions: read
+ packages: read
+ pages: read
+ repository-projects: read
+ security-events: read
+ statuses: read
+
+env:
+ GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+ GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+ beam_PostCommit_Python_Arm:
+ name: ${{matrix.job_name}} (${{matrix.job_phrase}}
${{matrix.python_version}})
+ runs-on: [self-hosted, ubuntu-20.04, main]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ job_name: [beam_PostCommit_Python_Arm]
+ job_phrase: [Run Python PostCommit Arm]
+ python_version: ['3.8', '3.9', '3.10', '3.11']
+ if: |
+ github.event_name == 'workflow_dispatch' ||
+ github.event_name == 'schedule' ||
+ startsWith(github.event.comment.body, 'Run Python PostCommit Arm')
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup repository
+ uses: ./.github/actions/setup-action
+ with:
+ comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{
matrix.python_version }})
+ - name: Install Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{matrix.python_version}}
+ - name: Install docker compose
+ run: |
+ sudo curl -L
https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname
-s)-$(uname -m) -o /usr/local/bin/docker-compose
+ sudo chmod +x /usr/local/bin/docker-compose
+ - name: Set PY_VER_CLEAN
+ id: set_py_ver_clean
+ run: |
+ PY_VER=${{ matrix.python_version }}
+ PY_VER_CLEAN=${PY_VER//.}
+ echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT
+ - name: run PostCommit Python ${{ matrix.python_version }} script
+ uses: ./.github/actions/gradle-command-self-hosted-action
+ with:
+ gradle-command:
:sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:postCommitArmIT
+ arguments: |
+ -PuseWheelDistribution \
+ -PpythonVersion=${{ matrix.python_version }} \
+ env:
+ CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
+ - name: Archive code coverage results
+ uses: actions/upload-artifact@v3
+ with:
+ name: python-code-coverage-report
+ path: "**/pytest*.xml"
diff --git a/sdks/python/scripts/run_integration_test.sh
b/sdks/python/scripts/run_integration_test.sh
index 4f29ed5a4ad..6ad592080ae 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -78,6 +78,7 @@
KMS_KEY_NAME="projects/apache-beam-testing/locations/global/keyRings/beam-it/cry
SUITE=""
COLLECT_MARKERS=
REQUIREMENTS_FILE=""
+ARCH=""
# Default test (pytest) options.
# Run WordCountIT.test_wordcount_it by default if no test options are
@@ -163,6 +164,11 @@ case $key in
shift # past argument
shift # past value
;;
+ --arch)
+ ARCH="$2"
+ shift # past argument
+ shift # past value
+ ;;
*) # unknown option
echo "Unknown option: $1"
exit 1
@@ -234,6 +240,10 @@ if [[ -z $PIPELINE_OPTS ]]; then
opts+=("--streaming")
fi
+ if [[ "$ARCH" == "ARM" ]]; then
+ opts+=("--machine_type=t2a-standard-1")
+ fi
+
if [[ ! -z "$KMS_KEY_NAME" ]]; then
opts+=(
"--kms_key_name=$KMS_KEY_NAME"
diff --git a/sdks/python/test-suites/dataflow/common.gradle
b/sdks/python/test-suites/dataflow/common.gradle
index f907a728c0b..7766cf3a377 100644
--- a/sdks/python/test-suites/dataflow/common.gradle
+++ b/sdks/python/test-suites/dataflow/common.gradle
@@ -143,6 +143,26 @@ task postCommitIT {
}
}
+task postCommitArmIT {
+ dependsOn 'initializeForDataflowJob'
+
+ doLast {
+ def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"]
+ def argMap = [
+ "test_opts": testOpts,
+ "sdk_location": project.ext.sdkLocation,
+ "suite": "postCommitIT-df${pythonVersionSuffix}",
+ "collect": "it_postcommit",
+ "arch": "ARM"
+ ]
+ def cmdArgs = mapToArgString(argMap)
+ exec {
+ executable 'sh'
+ args '-c', ". ${envdir}/bin/activate &&
${runScriptsDir}/run_integration_test.sh $cmdArgs"
+ }
+ }
+}
+
task postCommitSickbay {
dependsOn 'initializeForDataflowJob'
