This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new b5358c30d42 Fix precommit flink container (#34250)
b5358c30d42 is described below
commit b5358c30d4290efba4e9cb0a61ad3c3f49569eaa
Author: akashorabek <[email protected]>
AuthorDate: Wed Mar 19 19:33:04 2025 +0500
Fix precommit flink container (#34250)
* Fix Precommit Flink Container
* Fix image tag in the cleanup step
* Add beam-sdk-pr to public repos
---
.../workflows/beam_PreCommit_Flink_Container.yml | 33 ++++++++++++++++++++--
.../python_Combine_Flink_Batch_small.txt | 1 -
.../tools/stale_dataflow_prebuilt_image_cleaner.sh | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
4 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/beam_PreCommit_Flink_Container.yml
b/.github/workflows/beam_PreCommit_Flink_Container.yml
index d44f3a73092..e29ba956991 100644
--- a/.github/workflows/beam_PreCommit_Flink_Container.yml
+++ b/.github/workflows/beam_PreCommit_Flink_Container.yml
@@ -77,6 +77,10 @@ env:
HARNESS_IMAGES_TO_PULL:
gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
JOB_SERVER_IMAGE:
gcr.io/apache-beam-testing/beam_portability/beam_flink1.17_job_server:latest
ARTIFACTS_DIR: gs://beam-flink-cluster/beam-precommit-flink-container-${{
github.run_id }}
+ DOCKER_REGISTRY: gcr.io
+ DOCKER_REPOSITORY_ROOT: ${{ github.event_name == 'pull_request_target' &&
'gcr.io/apache-beam-testing/beam-sdk-pr' ||
'gcr.io/apache-beam-testing/beam-sdk' }}
+ PYTHON_VERSION: 3.9
+ PYTHON_SDK_IMAGE_TAG: latest
jobs:
beam_PreCommit_Flink_Container:
@@ -87,7 +91,7 @@ jobs:
github.event_name == 'pull_request_target' ||
github.event.comment.body == 'Run Flink Container PreCommit'
runs-on: [self-hosted, ubuntu-20.04, main]
- timeout-minutes: 45
+ timeout-minutes: 90
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
@@ -105,6 +109,24 @@ jobs:
uses: ./.github/actions/setup-environment-action
with:
python-version: default
+ - name: GCloud Docker credential helper
+ if: ${{ github.event_name == 'pull_request_target' }}
+ run: |
+ gcloud auth configure-docker ${{ env.DOCKER_REGISTRY }}
+ - name: Set PYTHON_SDK_IMAGE_TAG unique variable based on timestamp
+ if: ${{ github.event_name == 'pull_request_target' }}
+ run: echo "PYTHON_SDK_IMAGE_TAG=$(date +'%Y%m%d-%H%M%S%N')" >>
$GITHUB_ENV
+ - name: Build and push to registry
+ if: ${{ github.event_name == 'pull_request_target' }}
+ uses: ./.github/actions/gradle-command-self-hosted-action
+ with:
+ gradle-command: :sdks:python:container:py39:docker
+ arguments: |
+ -PpythonVersion=${{ env.PYTHON_VERSION }} \
+ -Pdocker-repository-root=${{ env.DOCKER_REPOSITORY_ROOT }} \
+ -Pdocker-tag=${{ env.PYTHON_SDK_IMAGE_TAG }} \
+ -PuseBuildx \
+ -Ppush-containers
- name: Prepare test arguments
uses: ./.github/actions/test-arguments-action
with:
@@ -141,11 +163,11 @@ jobs:
arguments: |
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
-Prunner=FlinkRunner \
- '-PloadTest.args=${{
env.beam_PreCommit_Flink_Container_test_arguments_2 }}
--job_name=flink-tests-python-${{env.NOW_UTC}}'
+ '-PloadTest.args=${{
env.beam_PreCommit_Flink_Container_test_arguments_2 }} --environment_config=${{
env.DOCKER_REPOSITORY_ROOT }}/beam_python${{ env.PYTHON_VERSION }}_sdk:${{
env.PYTHON_SDK_IMAGE_TAG }} --job_name=flink-tests-python-${{env.NOW_UTC}}'
# Run a Java Combine load test to verify the Flink container
- name: Run Flink Container Test with Java Combine
- timeout-minutes: 10
+ timeout-minutes: 20
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:java:testing:load-tests:run
@@ -158,3 +180,8 @@ jobs:
if: always()
run: |
${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete
+
+ - name: Cleanup Python SDK Container
+ if: ${{ always() && github.event_name == 'pull_request_target' }}
+ run: |
+ gcloud container images delete ${{ env.DOCKER_REPOSITORY_ROOT
}}/beam_python${{ env.PYTHON_VERSION }}_sdk:${{ env.PYTHON_SDK_IMAGE_TAG }}
--force-delete-tags --quiet
diff --git
a/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt
b/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt
index 5522a8f9b82..8c2bd838e81 100644
---
a/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt
+++
b/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt
@@ -18,6 +18,5 @@
--parallelism=2
--job_endpoint=localhost:8099
--environment_type=DOCKER
---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest
--top_count=10
--runner=PortableRunner
\ No newline at end of file
diff --git a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
index 21181a9a192..ed03ad6500c 100755
--- a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
+++ b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
@@ -22,7 +22,7 @@ set -euo pipefail
# Clean up private registry (us.gcr.io)
# Images more than 5 day old and not the latest (either has latest label or
newest)
-PUBLIC_REPOSITORIES=(beam-sdk beam_portability beamgrafana
beammetricssyncjenkins beammetricssyncgithub)
+PUBLIC_REPOSITORIES=(beam-sdk beam-sdk-pr beam_portability beamgrafana
beammetricssyncjenkins beammetricssyncgithub)
PRIVATE_REPOSITORIES=(java-postcommit-it python-postcommit-it jenkins
github-actions)
# set as the same as 6-week release period
if [[ $OSTYPE == "linux-gnu"* ]]; then
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 10135c842aa..51ed48738b3 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -573,7 +573,7 @@ class BeamModulePlugin implements Plugin<Project> {
}
project.ext.useBuildx = {
- return project.containerArchitectures() != [project.nativeArchitecture()]
+ return (project.containerArchitectures() !=
[project.nativeArchitecture()]) || project.rootProject.hasProperty("useBuildx")
}
/**
***********************************************************************************************/
