This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 9485109a4e1 [FLINK-40224][ci] Move common scripts out of
tools/azure-pipelines
9485109a4e1 is described below
commit 9485109a4e1eb5b835b99132c5b29a656bed29dc
Author: Dale Lane <[email protected]>
AuthorDate: Sat Jul 25 21:29:25 2026 +0100
[FLINK-40224][ci] Move common scripts out of tools/azure-pipelines
GHA pipelines use some of the assets from tools/azure-pipelines
This commit moves them into tools/ci to reflect that they are not
unique to the AZP builds, and are common across both.
Signed-off-by: Dale Lane <[email protected]>
---
.github/actions/job_init/action.yml | 4 ++--
.github/workflows/template.flink-ci.yml | 22 +++++++++++-----------
tools/azure-pipelines/e2e-template.yml | 8 ++++----
tools/azure-pipelines/jobs-template.yml | 14 +++++++-------
.../{azure-pipelines => ci}/cache_docker_images.sh | 0
.../create_build_artifact.sh | 0
tools/{azure-pipelines => ci}/debug_files_utils.sh | 0
tools/{azure-pipelines => ci}/free_disk_space.sh | 0
.../unpack_build_artifact.sh | 0
.../{azure-pipelines => ci}/uploading_watchdog.sh | 2 +-
10 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/.github/actions/job_init/action.yml
b/.github/actions/job_init/action.yml
index ea44331507d..eb9bb427e45 100644
--- a/.github/actions/job_init/action.yml
+++ b/.github/actions/job_init/action.yml
@@ -35,7 +35,7 @@ inputs:
required: false
default: ""
free_disk_space:
- description: "Runs tools/azure-pipelines/free_disk_space.sh to remove
unneeded pre-installed software from the runner (only works for jobs that run
directly on the host, not in a container)."
+ description: "Runs tools/ci/free_disk_space.sh to remove unneeded
pre-installed software from the runner (only works for jobs that run directly
on the host, not in a container)."
required: false
default: "false"
outputs:
@@ -62,7 +62,7 @@ runs:
- name: "Free up disk space"
if: ${{ inputs.free_disk_space == 'true' }}
shell: bash
- run: ./tools/azure-pipelines/free_disk_space.sh
+ run: ./tools/ci/free_disk_space.sh
- name: "Delete unused binaries"
if: ${{ inputs.free_disk_space != 'true' }}
diff --git a/.github/workflows/template.flink-ci.yml
b/.github/workflows/template.flink-ci.yml
index 38fde303633..02e5ae33d35 100644
--- a/.github/workflows/template.flink-ci.yml
+++ b/.github/workflows/template.flink-ci.yml
@@ -64,7 +64,7 @@ env:
MAVEN_REPO_FOLDER: /root/.m2/repository
MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository
- # required by tools/azure-pipelines/cache_docker_images.sh
+ # required by tools/ci/cache_docker_images.sh
DOCKER_IMAGES_CACHE_FOLDER: /root/.docker-cache
jobs:
@@ -113,7 +113,7 @@ jobs:
- name: "Collect build artifacts"
working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
run: |
- ./tools/azure-pipelines/create_build_artifact.sh
+ ./tools/ci/create_build_artifact.sh
- name: "Upload artifacts to make them available in downstream jobs"
uses: actions/upload-artifact@v7
@@ -172,7 +172,7 @@ jobs:
- name: "Unpack build artifact"
working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
run: |
- ./tools/azure-pipelines/unpack_build_artifact.sh
+ ./tools/ci/unpack_build_artifact.sh
- name: "Test"
working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
@@ -261,7 +261,7 @@ jobs:
- name: "Unpack build artifact"
working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
run: |
- ./tools/azure-pipelines/unpack_build_artifact.sh
+ ./tools/ci/unpack_build_artifact.sh
- name: "Try loading Docker images from Cache"
id: docker-cache
@@ -274,7 +274,7 @@ jobs:
- name: "Load Docker images if not present in cache, yet"
if: ${{ !cancelled() && steps.docker-cache.outputs.cache-hit != 'true'
}}
working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
- run: ./tools/azure-pipelines/cache_docker_images.sh load
+ run: ./tools/ci/cache_docker_images.sh load
- name: "Create non-root user for test execution"
run: |
@@ -304,7 +304,7 @@ jobs:
DOCKER_HOST: unix:///var/run/docker.sock
timeout-minutes: ${{ fromJSON(env.GHA_JOB_TIMEOUT) }}
run: |
- runuser -u flink -- bash -c '${{ inputs.environment }}
PROFILE="$PROFILE -Pgithub-actions"
./tools/azure-pipelines/uploading_watchdog.sh \
+ runuser -u flink -- bash -c '${{ inputs.environment }}
PROFILE="$PROFILE -Pgithub-actions" ./tools/ci/uploading_watchdog.sh \
./tools/ci/test_controller.sh ${{ matrix.module }}'
- name: "Post-build Disk Info"
@@ -332,7 +332,7 @@ jobs:
- name: "Save Docker images to cache"
working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
if: ${{ !cancelled() && (failure() ||
steps.docker-cache.outputs.cache-hit != 'true') }}
- run: ./tools/azure-pipelines/cache_docker_images.sh save
+ run: ./tools/ci/cache_docker_images.sh save
- name: "Save Maven package cache"
if: ${{ !cancelled() && steps.job-init.outputs.maven-cache-hit !=
'true' }}
@@ -412,7 +412,7 @@ jobs:
path: ${{ env.FLINK_ARTIFACT_DIR }}
- name: "Unpack build artifact"
- run: ./tools/azure-pipelines/unpack_build_artifact.sh
+ run: ./tools/ci/unpack_build_artifact.sh
# the cache task does not create directories on a cache miss, and can
later fail when trying to tar the directory if the test haven't created it
# this may for example happen if a given directory is only used by a
subset of tests, which are run in a different 'group'
@@ -444,7 +444,7 @@ jobs:
- name: "Load Docker images if not present in Cache, yet"
if: ${{ !cancelled() && steps.docker-cache.outputs.cache-hit != 'true'
}}
- run: ./tools/azure-pipelines/cache_docker_images.sh load
+ run: ./tools/ci/cache_docker_images.sh load
- name: "Build Flink"
uses: "./.github/actions/run_mvn"
@@ -460,7 +460,7 @@ jobs:
IT_CASE_S3_SECRET_KEY: ${{ secrets.s3_secret_key }}
timeout-minutes: ${{ fromJSON(env.GHA_JOB_TIMEOUT) }}
run: |
- ${{ inputs.environment }} FLINK_DIR=`pwd`/build-target
PROFILE="$PROFILE -Pgithub-actions"
./tools/azure-pipelines/uploading_watchdog.sh \
+ ${{ inputs.environment }} FLINK_DIR=`pwd`/build-target
PROFILE="$PROFILE -Pgithub-actions" ./tools/ci/uploading_watchdog.sh \
flink-end-to-end-tests/run-nightly-tests.sh ${{ matrix.group }}
- name: "Upload Logs"
@@ -472,7 +472,7 @@ jobs:
- name: "Save Docker images to Cache"
if: ${{ !cancelled() && (failure() ||
steps.docker-cache.outputs.cache-hit != 'true') }}
- run: ./tools/azure-pipelines/cache_docker_images.sh save
+ run: ./tools/ci/cache_docker_images.sh save
- name: "Save Maven package cache"
if: ${{ !cancelled() && steps.job-init.outputs.maven-cache-hit !=
'true' }}
diff --git a/tools/azure-pipelines/e2e-template.yml
b/tools/azure-pipelines/e2e-template.yml
index 3b1229af1a0..8045525156c 100644
--- a/tools/azure-pipelines/e2e-template.yml
+++ b/tools/azure-pipelines/e2e-template.yml
@@ -45,7 +45,7 @@ jobs:
fi
displayName: Check if Docs only PR
# free up disk space before running anything caching related. Caching has
proven to fail in the past, due to lacking disk space.
- - script: ./tools/azure-pipelines/free_disk_space.sh
+ - script: ./tools/ci/free_disk_space.sh
target: host
displayName: Free up disk space
- task: UsePythonVersion@0
@@ -101,7 +101,7 @@ jobs:
continueOnError: true
condition: not(eq(variables['SKIP'], '1'))
displayName: Cache docker images
- - script: ./tools/azure-pipelines/cache_docker_images.sh load
+ - script: ./tools/ci/cache_docker_images.sh load
displayName: Restore docker images
condition: and(not(canceled()), eq(variables.DOCKER_IMAGES_CACHE_HIT,
'true'))
continueOnError: true
@@ -131,7 +131,7 @@ jobs:
- script: ${{parameters.environment}} PROFILE="$PROFILE -Dfast
-Pskip-webui-build" ./tools/ci/compile_ci.sh
displayName: Build Flink
condition: not(eq(variables['SKIP'], '1'))
- - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target
./tools/azure-pipelines/uploading_watchdog.sh
flink-end-to-end-tests/run-nightly-tests.sh ${{parameters.group}}
+ - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target
./tools/ci/uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh
${{parameters.group}}
displayName: Run e2e tests
env:
IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)
@@ -145,7 +145,7 @@ jobs:
inputs:
targetPath: $(DEBUG_FILES_OUTPUT_DIR)
artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME)
- - script: ./tools/azure-pipelines/cache_docker_images.sh save
+ - script: ./tools/ci/cache_docker_images.sh save
displayName: Save docker images
condition: and(not(eq(variables['SKIP'], '1')),
ne(variables.DOCKER_IMAGES_CACHE_HIT, 'true'))
continueOnError: true
diff --git a/tools/azure-pipelines/jobs-template.yml
b/tools/azure-pipelines/jobs-template.yml
index a9d97ba4365..d86b5210465 100644
--- a/tools/azure-pipelines/jobs-template.yml
+++ b/tools/azure-pipelines/jobs-template.yml
@@ -37,7 +37,7 @@ jobs:
steps:
# if on Azure, free up disk space
- - script: ./tools/azure-pipelines/free_disk_space.sh
+ - script: ./tools/ci/free_disk_space.sh
target: host
condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
displayName: Free up disk space
@@ -69,7 +69,7 @@ jobs:
# Compile
- script: |
${{parameters.environment}} ./tools/ci/compile_ci.sh || exit $?
- ./tools/azure-pipelines/create_build_artifact.sh
+ ./tools/ci/create_build_artifact.sh
displayName: Compile
# upload artifacts for next stage
@@ -103,7 +103,7 @@ jobs:
module: misc
steps:
# if on Azure, free up disk space
- - script: ./tools/azure-pipelines/free_disk_space.sh
+ - script: ./tools/ci/free_disk_space.sh
target: host
condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
displayName: Free up disk space
@@ -121,7 +121,7 @@ jobs:
path: $(FLINK_ARTIFACT_DIR)
artifact: FlinkCompileArtifact-${{parameters.stage_name}}
- - script: ./tools/azure-pipelines/unpack_build_artifact.sh
+ - script: ./tools/ci/unpack_build_artifact.sh
displayName: "Unpack Build artifact"
- task: Cache@2
@@ -142,7 +142,7 @@ jobs:
condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
displayName: Cache docker images
- - script: ./tools/azure-pipelines/cache_docker_images.sh load
+ - script: ./tools/ci/cache_docker_images.sh load
displayName: Restore docker images
condition: and(not(canceled()), eq(variables.DOCKER_IMAGES_CACHE_HIT,
'true'))
continueOnError: true
@@ -156,7 +156,7 @@ jobs:
displayName: Set coredump pattern
# Test
- - script: ${{parameters.environment}}
./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh
$(module)
+ - script: ${{parameters.environment}} ./tools/ci/uploading_watchdog.sh
./tools/ci/test_controller.sh $(module)
displayName: Test - $(module)
env:
IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)
@@ -176,7 +176,7 @@ jobs:
targetPath: $(DEBUG_FILES_OUTPUT_DIR)
artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME)
- - script: ./tools/azure-pipelines/cache_docker_images.sh save
+ - script: ./tools/ci/cache_docker_images.sh save
displayName: Save docker images
condition: and(not(canceled()), or(failed(),
ne(variables.DOCKER_IMAGES_CACHE_HIT, 'true')))
continueOnError: true
diff --git a/tools/azure-pipelines/cache_docker_images.sh
b/tools/ci/cache_docker_images.sh
similarity index 100%
rename from tools/azure-pipelines/cache_docker_images.sh
rename to tools/ci/cache_docker_images.sh
diff --git a/tools/azure-pipelines/create_build_artifact.sh
b/tools/ci/create_build_artifact.sh
similarity index 100%
rename from tools/azure-pipelines/create_build_artifact.sh
rename to tools/ci/create_build_artifact.sh
diff --git a/tools/azure-pipelines/debug_files_utils.sh
b/tools/ci/debug_files_utils.sh
similarity index 100%
rename from tools/azure-pipelines/debug_files_utils.sh
rename to tools/ci/debug_files_utils.sh
diff --git a/tools/azure-pipelines/free_disk_space.sh
b/tools/ci/free_disk_space.sh
similarity index 100%
rename from tools/azure-pipelines/free_disk_space.sh
rename to tools/ci/free_disk_space.sh
diff --git a/tools/azure-pipelines/unpack_build_artifact.sh
b/tools/ci/unpack_build_artifact.sh
similarity index 100%
rename from tools/azure-pipelines/unpack_build_artifact.sh
rename to tools/ci/unpack_build_artifact.sh
diff --git a/tools/azure-pipelines/uploading_watchdog.sh
b/tools/ci/uploading_watchdog.sh
similarity index 98%
rename from tools/azure-pipelines/uploading_watchdog.sh
rename to tools/ci/uploading_watchdog.sh
index 5b5963782f4..cce732013ea 100755
--- a/tools/azure-pipelines/uploading_watchdog.sh
+++ b/tools/ci/uploading_watchdog.sh
@@ -58,7 +58,7 @@ fi
source "${HERE}/../ci/controller_utils.sh"
-source ./tools/azure-pipelines/debug_files_utils.sh
+source ./tools/ci/debug_files_utils.sh
prepare_debug_files "${temporary_folder}" "${job_name}"
export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs"
mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory
'${FLINK_LOG_DIR}'." ; exit 1; }
