gopidesupavan commented on code in PR #45266:
URL: https://github.com/apache/airflow/pull/45266#discussion_r1899169669
##########
.github/workflows/ci-image-build.yml:
##########
@@ -104,27 +97,17 @@ jobs:
strategy:
fail-fast: true
matrix:
- # yamllint disable-line rule:line-length
- python-version: ${{ inputs.do-build == 'true' &&
fromJSON(inputs.python-versions) || fromJSON('[""]') }}
+ python-version: ${{ fromJSON(inputs.python-versions) ||
fromJSON('[""]') }}
timeout-minutes: 110
- name: "\
-${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
-CI ${{ inputs.platform }} image\
-${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
-${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
- # The ARM images need to be built using self-hosted runners as ARM macos
public runners
- # do not yet allow us to run docker effectively and fast.
- #
https://github.com/actions/runner-images/issues/9254#issuecomment-1917916016
- # https://github.com/abiosoft/colima/issues/970
- # https://github.com/actions/runner/issues/1456
- # See https://github.com/apache/airflow/pull/38640
+ name: "Build CI ${{ inputs.platform }} image ${{ matrix.python-version }}"
# NOTE!!!!! This has to be put in one line for runs-on to recognize the
"fromJSON" properly !!!!
# adding space before (with >) apparently turns the `runs-on` processed
line into a string "Array"
# instead of an array of strings.
# yamllint disable-line rule:line-length
- runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(inputs.runs-on-as-json-public) ||
fromJSON(inputs.runs-on-as-json-self-hosted) }}
+ runs-on: ${{ (inputs.platform == 'linuz/amd64') &&
fromJSON(inputs.runs-on-as-json-public) ||
fromJSON(inputs.runs-on-as-json-self-hosted) }}
Review Comment:
```suggestion
runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(inputs.runs-on-as-json-public) ||
fromJSON(inputs.runs-on-as-json-self-hosted) }}
```
##########
.github/workflows/ci-image-build.yml:
##########
@@ -137,42 +120,32 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || ''
}}"
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: inputs.do-build == 'true'
- name: "Checkout target branch"
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Checkout target commit"
- uses: ./.github/actions/checkout_target_commit
- if: inputs.do-build == 'true'
- with:
- target-commit-sha: ${{ inputs.target-commit-sha }}
- pull-request-target: ${{ inputs.pull-request-target }}
- is-committer-build: ${{ inputs.is-committer-build }}
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
- if: inputs.do-build == 'true'
- name: "Install Breeze"
uses: ./.github/actions/breeze
- if: inputs.do-build == 'true'
- - name: "Regenerate dependencies in case they were modified manually so
that we can build an image"
+ - name: "Restore CI docker image ${{ inputs.platform }}:${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
+ uses:
apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+ with:
+ key: "ci-image-save-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
+ path: "/tmp/"
+ id: restore-ci-image
+ - name: "Load CI image ${{ inputs.platform }}:${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
+ run: breeze ci-image load --platform ${{ inputs.platform }}
shell: bash
- run: |
- pip install rich>=12.4.4 pyyaml
- python scripts/ci/pre_commit/update_providers_dependencies.py
- if: inputs.do-build == 'true' && inputs.upgrade-to-newer-dependencies
!= 'false'
- - name: "Start ARM instance"
- run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: inputs.do-build == 'true' && inputs.platform == 'linux/arm64'
- - name: Login to ghcr.io
- run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- if: inputs.do-build == 'true'
+ if: steps.restore-ci-image.stash-hit == 'true'
- name: >
Build ${{ inputs.push-image == 'true' && ' & push ' || '' }}
- ${{ inputs.platform }}:${{ matrix.python-version }}:${{
inputs.image-tag }}
+ ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} image
run: >
- breeze ci-image build --builder airflow_cache --tag-as-latest
--image-tag "${{ inputs.image-tag }}"
- --python "${{ matrix.python-version }}" --platform "${{
inputs.platform }}"
+ breeze ci-image build
+ --docker-cache local
Review Comment:
so now cache is always local, no cache push to ghcr?
##########
.github/workflows/ci-image-build.yml:
##########
@@ -137,42 +120,32 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || ''
}}"
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: inputs.do-build == 'true'
- name: "Checkout target branch"
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Checkout target commit"
- uses: ./.github/actions/checkout_target_commit
- if: inputs.do-build == 'true'
- with:
- target-commit-sha: ${{ inputs.target-commit-sha }}
- pull-request-target: ${{ inputs.pull-request-target }}
- is-committer-build: ${{ inputs.is-committer-build }}
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
- if: inputs.do-build == 'true'
- name: "Install Breeze"
uses: ./.github/actions/breeze
- if: inputs.do-build == 'true'
- - name: "Regenerate dependencies in case they were modified manually so
that we can build an image"
+ - name: "Restore CI docker image ${{ inputs.platform }}:${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
+ uses:
apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+ with:
+ key: "ci-image-save-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
+ path: "/tmp/"
+ id: restore-ci-image
+ - name: "Load CI image ${{ inputs.platform }}:${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
+ run: breeze ci-image load --platform ${{ inputs.platform }}
Review Comment:
```suggestion
run: breeze ci-image load --platform ${{ inputs.platform }} --python
${{ env. PYTHON_MAJOR_MINOR_VERSION }}
```
##########
.github/workflows/ci-image-build.yml:
##########
@@ -189,7 +162,14 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
GITHUB_USERNAME: ${{ github.actor }}
PUSH: ${{ inputs.push-image }}
VERBOSE: "true"
- if: inputs.do-build == 'true'
- - name: "Stop ARM instance"
- run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && inputs.do-build == 'true' && inputs.platform ==
'linux/arm64'
+ - name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+ run: breeze ci-image save --platform "${{ inputs.platform }}"
Review Comment:
```suggestion
run: breeze ci-image save --platform "${{ inputs.platform }}"
--python ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
