This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2acc9ccba34 Reenable PreCommit Flink Container after Fix cleanup
scripts (#33957)
2acc9ccba34 is described below
commit 2acc9ccba342a84b54558256ef3cb3f48c4f0499
Author: Yi Hu <[email protected]>
AuthorDate: Tue Feb 11 17:41:49 2025 -0500
Reenable PreCommit Flink Container after Fix cleanup scripts (#33957)
* Revert "Disable PreCommit Flink Container (#33956)"
This reverts commit 7ac5bc43ecfdbb0b138484ae8d2f33bc9ddaaa0f.
* Fix clean up scripts
* Fix trigger paths
* Fix script
---
.github/autolabeler.yml | 2 +-
.github/workflows/beam_PreCommit_Flink_Container.yml | 19 ++++++++++++++++++-
.../workflows/beam_PreCommit_Java_GCP_IO_Direct.yml | 2 --
.github/workflows/beam_PreCommit_Portable_Python.yml | 4 ----
.github/workflows/beam_PreCommit_Python_PVR_Flink.yml | 4 ----
.test-infra/dataproc/cleanup.sh | 3 ++-
.../tools/stale_dataflow_prebuilt_image_cleaner.sh | 2 +-
7 files changed, 22 insertions(+), 14 deletions(-)
diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml
index a1a4c3bf039..eed6e34d09e 100644
--- a/.github/autolabeler.yml
+++ b/.github/autolabeler.yml
@@ -79,7 +79,7 @@ io: ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*",
"sdks/python/apache_beam/
# Runners
"runners": ["runners/**/*", "sdks/go/pkg/beam/runners/**/*",
"sdks/python/apache_beam/runners/**/*",
"sdks/typescript/src/apache_beam/runners/**/*"]
-"core": ["runners/core-construction-java/**/*", "runners/core-java/**/*"]
+"core": ["runners/core-java/**/*"]
"dataflow": ["runners/google-cloud-dataflow-java/**/*",
"sdks/go/pkg/beam/runners/dataflow/**/*", "sdks/python/runners/dataflow/**/*"]
"direct": ["runners/direct-java/**/*", "sdks/go/pkg/beam/runners/direct/**/*",
"sdks/python/runners/direct/**/*"]
"flink": ["runners/flink/**/*", "sdks/go/pkg/beam/runners/flink/**/*"]
diff --git a/.github/workflows/beam_PreCommit_Flink_Container.yml
b/.github/workflows/beam_PreCommit_Flink_Container.yml
index 817153b580d..d44f3a73092 100644
--- a/.github/workflows/beam_PreCommit_Flink_Container.yml
+++ b/.github/workflows/beam_PreCommit_Flink_Container.yml
@@ -17,10 +17,27 @@ name: PreCommit Flink Container
on:
pull_request_target:
- # TODO(https://github.com/apache/beam/pull/33956) re-enable when two
issues resolved.
paths:
+ - 'model/**'
+ - 'sdks/python/apache_beam/runners/portability/**'
+ - 'release/**'
+ - 'runners/core-java/**'
+ - 'runners/flink/**'
+ - 'runners/java-fn-execution/**'
- '.github/trigger_files/beam_PreCommit_Flink_Container.json'
- 'release/trigger_all_tests.json'
+ push:
+ tags: [ 'v*' ]
+ branches: [ 'master', 'release-*' ]
+ paths:
+ - 'model/**'
+ - 'sdks/python/apache_beam/runners/portability/**'
+ - 'release/**'
+ - 'runners/core-java/**'
+ - 'runners/flink/**'
+ - 'runners/java-fn-execution/**'
+ schedule:
+ - cron: '0 */6 * * *'
workflow_dispatch:
# Setting explicit permissions for the action to avoid the default permissions
which are `write-all`
diff --git a/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml
b/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml
index fbcf108791f..32ec1516195 100644
--- a/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml
@@ -20,7 +20,6 @@ on:
tags: ['v*']
branches: ['master', 'release-*']
paths:
- - "runners/core-construction-java/**"
- "runners/core-java/**"
- "sdks/java/core/src/main/**"
- "sdks/java/extensions/arrow/**"
@@ -34,7 +33,6 @@ on:
pull_request_target:
branches: ['master', 'release-*']
paths:
- - "runners/core-construction-java/**"
- "runners/core-java/**"
- 'release/trigger_all_tests.json'
- '.github/trigger_files/beam_PreCommit_Java_GCP_IO_Direct.json'
diff --git a/.github/workflows/beam_PreCommit_Portable_Python.yml
b/.github/workflows/beam_PreCommit_Portable_Python.yml
index 1b7ec5532b1..883294b1d58 100644
--- a/.github/workflows/beam_PreCommit_Portable_Python.yml
+++ b/.github/workflows/beam_PreCommit_Portable_Python.yml
@@ -21,12 +21,10 @@ on:
branches: ['master', 'release-*']
paths:
- 'model/**'
- - 'runners/core-construction-java/**'
- 'runners/core-java/**'
- 'runners/extensions-java/**'
- 'runners/flink/**'
- 'runners/java-fn-execution/**'
- - 'runners/reference/**'
- 'sdks/python/**'
- 'release/**'
- '.github/workflows/beam_PreCommit_Portable_Python.yml'
@@ -34,12 +32,10 @@ on:
branches: ['master', 'release-*']
paths:
- 'model/**'
- - 'runners/core-construction-java/**'
- 'runners/core-java/**'
- 'runners/extensions-java/**'
- 'runners/flink/**'
- 'runners/java-fn-execution/**'
- - 'runners/reference/**'
- 'sdks/python/**'
- 'release/**'
- 'release/trigger_all_tests.json'
diff --git a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml
b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml
index 50ec86e73b3..2010b2ff6f4 100644
--- a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml
+++ b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml
@@ -23,12 +23,10 @@ on:
- 'sdks/python/**'
- 'release/**'
- 'sdks/java/io/kafka/**'
- - 'runners/core-construction-java/**'
- 'runners/core-java/**'
- 'runners/extensions-java/**'
- 'runners/flink/**'
- 'runners/java-fn-execution/**'
- - 'runners/reference/**'
- 'release/trigger_all_tests.json'
- '.github/trigger_files/beam_PreCommit_Python_PVR_Flink.json'
issue_comment:
@@ -41,12 +39,10 @@ on:
- 'sdks/python/**'
- 'release/**'
- 'sdks/java/io/kafka/**'
- - 'runners/core-construction-java/**'
- 'runners/core-java/**'
- 'runners/extensions-java/**'
- 'runners/flink/**'
- 'runners/java-fn-execution/**'
- - 'runners/reference/**'
- '.github/workflows/beam_PreCommit_Python_PVR_Flink.yml'
schedule:
- cron: '45 2/6 * * *'
diff --git a/.test-infra/dataproc/cleanup.sh b/.test-infra/dataproc/cleanup.sh
index 95eea525a2e..41e2aff720f 100755
--- a/.test-infra/dataproc/cleanup.sh
+++ b/.test-infra/dataproc/cleanup.sh
@@ -21,7 +21,8 @@ toDeleteList=( )
generatedResources=("beam-loadtests-go-*-flink"
"beam-loadtests-python-*-flink" \
-"beam-loadtests-py-*-flink" "beam-postcommit-python-chicago" )
+"beam-loadtests-py-*-flink" "beam-postcommit-python-chicago" \
+"beam-precommit-flink-container" )
function deleteFilteredClusters(){
for cluster in ${toDeleteList[@]};do
diff --git a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
index e34f637dfbe..21181a9a192 100755
--- a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
+++ b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
@@ -96,7 +96,7 @@ for image_name in ${IMAGE_NAMES[@]}; do
# they will have a virtual size of 0 and a created date at the start
of the epoch, but their manifests will
# point to active images. These images should only be deleted when
all of their dependencies can be safely
# deleted.
- MANIFEST=$(docker manifest inspect ${image_name}@"${current}" ||
echo "")
+ MANIFEST=$(docker buildx imagetools inspect
${image_name}@"${current}" --raw || echo "")
if [ -z "$MANIFEST" ]; then
# Sometimes "no such manifest" seen. Skip current if command hit
error
FAILED_IMAGES+=" $current"
