This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 25fa309 Easy switching between GitHub Container Registries (#14120)
25fa309 is described below
commit 25fa3092323e0cd13b11bb8e809b88ff3c043e4e
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Feb 23 12:44:06 2021 +0100
Easy switching between GitHub Container Registries (#14120)
This change enables easy switching between GitHub Package Registry
and GitHub Container Registry by simply adding GITHUB_REGISTRY
secret to be either `docker.package.github.com` or `ghcr.io`.
This makes it easy to switch only by the Apache Airflow repository
run builds, as it requires preparation of images (to make them
public and to add permissions to manage them) after they got
created for the first time. GitHub Package Registry works
out-of-the-box but it is less stable and considered a legacy,
also it does not allow image retention.
Documentation has been updated to reflect the reasoning of choosing
this solution as well as describing maintenance processes around
images (including adding new Python version)
---
.github/workflows/build-images-workflow-run.yml | 3 +-
.github/workflows/ci.yml | 47 +++++++--
.github/workflows/scheduled_quarantined.yml | 5 +-
CI.rst | 124 +++++++++++++++++++++--
IMAGES.rst | 61 ++++++++++-
scripts/ci/images/ci_wait_for_all_prod_images.sh | 1 +
scripts/ci/libraries/_initialization.sh | 2 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 103 ++++++++++---------
8 files changed, 270 insertions(+), 76 deletions(-)
diff --git a/.github/workflows/build-images-workflow-run.yml
b/.github/workflows/build-images-workflow-run.yml
index b9873e7..e5f8b41 100644
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@@ -30,8 +30,6 @@ env:
DB_RESET: "true"
VERBOSE: "true"
USE_GITHUB_REGISTRY: "true"
- # Might be either 'ghcr.io' or 'docker.pkg.github.com'
- GITHUB_REGISTRY: "docker.pkg.github.com"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your
repo but by default the
@@ -47,6 +45,7 @@ env:
GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
BUILD_IMAGES: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false'
}}
INSTALL_PROVIDERS_FROM_SOURCES: "true"
+ GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }}
jobs:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6f4a472..e958aa4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -67,7 +67,6 @@ env:
#
# You can also switch back to building images locally and disabling the
"Build Images" workflow
# by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set
to "false"
-
GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{
secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
jobs:
@@ -203,6 +202,9 @@ jobs:
env:
BACKEND: sqlite
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgradeToNewerDependencies }}
+ WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }}
+ outputs:
+ githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -222,16 +224,20 @@ jobs:
- name: >
Wait for CI images
${{ needs.build-info.outputs.pythonVersions }}:${{
env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
+ id: wait-for-images
env:
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
# We wait for the images to be available either from the
build-ci-image step or from
- # "build-images-workflow-run.yml' run as pull_request_target (it has
the write
- # permissions in case pull_request from fork is run.
+ # "build-images-workflow-run.yml' run as pull_request_target.
# We are utilising single job to wait for all images because this job
merely waits
- # For the images to be available. The test jobs wait for it to
complete!
+ # for the images to be available.
+ # The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'!
+ # The job will set the output "githubRegistry" - result of auto-detect
which registry has
+ # been used by checking where the image can be downloaded from.
+ #
run: ./scripts/ci/images/ci_wait_for_all_ci_images.sh
- if: needs.build-info.outputs.waitForImage == 'true'
+
verify-ci-images:
timeout-minutes: 20
@@ -243,6 +249,7 @@ jobs:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions)
}}
env:
BACKEND: sqlite
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -270,6 +277,7 @@ jobs:
SKIP: "pylint,identity"
MOUNT_SELECTED_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION:
${{needs.build-info.outputs.defaultPythonVersion}}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.basic-checks-only == 'false'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -360,6 +368,7 @@ jobs:
# to the image but we want to static-check all of them
MOUNT_SELECTED_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION:
${{needs.build-info.outputs.defaultPythonVersion}}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -397,6 +406,8 @@ jobs:
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, ci-images]
if: needs.build-info.outputs.docs-build == 'true'
+ env:
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -447,6 +458,7 @@ jobs:
VERSION_SUFFIX_FOR_PYPI: "dev"
VERSION_SUFFIX_FOR_SVN: "dev"
PACKAGE_FORMAT: ${{ matrix.package-format }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -497,6 +509,7 @@ jobs:
VERSION_SUFFIX_FOR_PYPI: "dev"
VERSION_SUFFIX_FOR_SVN: "dev"
PACKAGE_FORMAT: ${{ matrix.package-format }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
strategy:
matrix:
package-format: ['wheel', 'sdist']
@@ -543,6 +556,7 @@ jobs:
VERSION_SUFFIX_FOR_PYPI: "dev"
VERSION_SUFFIX_FOR_SVN: "dev"
PACKAGE_FORMAT: ${{ matrix.package-format }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
strategy:
matrix:
package-format: ['wheel', 'sdist']
@@ -579,6 +593,7 @@ jobs:
TEST_TYPES: "Helm"
BACKEND: "sqlite"
PYTHON_MAJOR_MINOR_VERSION:
${{needs.build-info.outputs.defaultPythonVersion}}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: >
needs.build-info.outputs.needs-helm-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name !=
'schedule')
@@ -640,6 +655,7 @@ jobs:
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -698,6 +714,7 @@ jobs:
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -753,6 +770,7 @@ jobs:
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -813,6 +831,7 @@ jobs:
TEST_TYPE: ""
NUM_RUNS: 10
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -904,11 +923,13 @@ jobs:
name: "Wait for PROD images"
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, ci-images]
+ if: needs.build-info.outputs.image-build == 'true'
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{
needs.build-info.outputs.defaultPythonVersion }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgradeToNewerDependencies }}
- if: needs.build-info.outputs.image-build == 'true'
+ outputs:
+ githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -928,11 +949,18 @@ jobs:
- name: >
Wait for PROD images
${{ needs.build-info.outputs.pythonVersions }}:${{
env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
+ # We wait for the images to be available either from the
build-ci-image step or from
+ # "build-images-workflow-run.yml' run as pull_request_target.
+ # We are utilising single job to wait for all images because this job
merely waits
+ # For the images to be available. The test jobs wait for it to
complete!
+ # The job will set the output "githubRegistry" - result of auto-detect
which registry has
+ # been used by checking where the image can be downloaded from.
+ #
+ id: wait-for-images
env:
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
run: ./scripts/ci/images/ci_wait_for_all_prod_images.sh
- if: needs.build-info.outputs.waitForImage == 'true'
verify-prod-images:
timeout-minutes: 20
@@ -944,6 +972,7 @@ jobs:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions)
}}
env:
BACKEND: sqlite
+ GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -982,6 +1011,7 @@ jobs:
KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
+ GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-kubernetes-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -1063,6 +1093,7 @@ jobs:
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
+ GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -1111,6 +1142,7 @@ jobs:
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -1141,6 +1173,7 @@ jobs:
- ci-images
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
# Only run it for direct pushes
if: >
github.ref == 'refs/heads/master' || github.ref ==
'refs/heads/v1-10-test' ||
diff --git a/.github/workflows/scheduled_quarantined.yml
b/.github/workflows/scheduled_quarantined.yml
index 889cdb7..00663cb 100644
--- a/.github/workflows/scheduled_quarantined.yml
+++ b/.github/workflows/scheduled_quarantined.yml
@@ -33,8 +33,6 @@ env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
PYTHON_MAJOR_MINOR_VERSION: 3.6
USE_GITHUB_REGISTRY: "true"
- # Might be either 'ghcr.io' or 'docker.pkg.github.com'
- GITHUB_REGISTRY: "docker.pkg.github.com"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# This token is WRITE one - schedule type of events always have the WRITE
token
@@ -46,6 +44,7 @@ env:
GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
+ GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }}
jobs:
@@ -88,6 +87,8 @@ jobs:
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+ env:
+ GITHUB_REGISTRY: ${{
steps.determine-github-registry.outputs.githubRegistry }}
- name: "Tests"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- uses: actions/upload-artifact@v2
diff --git a/CI.rst b/CI.rst
index 850839b..0c8e03b 100644
--- a/CI.rst
+++ b/CI.rst
@@ -33,8 +33,6 @@ environments we use. Most of our CI jobs are written as bash
scripts which are e
the CI jobs. And we have a number of variables determine build behaviour.
-
-
GitHub Actions runs
-------------------
@@ -53,14 +51,23 @@ techniques have been implemented that use efficiently cache
from the GitHub Dock
this brings down the time needed to rebuild the image to ~4 minutes. In some
cases (when dependencies change)
it can be ~6-7 minutes and in case base image of Python releases new
patch-level, it can be ~12 minutes.
+Container Registry used as cache
+--------------------------------
+
+For the CI builds of our we are using Container Registry to store results of
the "Build Image" workflow
+and pass it to the "CI Build" workflow.
+
Currently in master version of Airflow we run tests in 3 different versions of
Python (3.6, 3.7, 3.8)
which means that we have to build 6 images (3 CI ones and 3 PROD ones). Yet we
run around 12 jobs
with each of the CI images. That is a lot of time to just build the
environment to run. Therefore
-we are utilising ``workflow_run`` feature of GitHub Actions. This feature
allows to run a separate,
-independent workflow, when the main workflow is run - this separate workflow
is different than the main
-one, because by default it runs using ``master`` version of the sources but
also - and most of all - that
-it has WRITE access to the repository. This is especially important in our
case where Pull Requests to
-Airflow might come from any repository, and it would be a huge security issue
if anyone from outside could
+we are utilising ``workflow_run`` feature of GitHub Actions.
+
+This feature allows to run a separate, independent workflow, when the main
workflow is run -
+this separate workflow is different than the main one, because by default it
runs using ``master`` version
+of the sources but also - and most of all - that it has WRITE access to the
repository.
+
+This is especially important in our case where Pull Requests to Airflow might
come from any repository,
+and it would be a huge security issue if anyone from outside could
utilise the WRITE access to Apache Airflow repository via an external Pull
Request.
Thanks to the WRITE access and fact that the 'workflow_run' by default uses
the 'master' version of the
@@ -71,9 +78,56 @@ this image can be built only once and used by all the jobs
running tests. The im
rather than build it from the scratch. Pulling such image takes ~ 1 minute,
thanks to that we are saving
a lot of precious time for jobs.
-
-Local runs
-----------
+We can use either of the two available GitHub Container registries as cache:
+
+* Legacy `GitHub Package Registry <https://github.com/features/packages>`_
which is not very
+ stable, uses old infrastructure of GitHub and it lacks certain features -
notably it does not allow
+ us to delete the old image. The benefit of using GitHub Package Registry is
that it works
+ out-of-the-box (write authentication is done using ``GITHUB_TOKEN`` and
users do not have to do any
+ action to make it work in case they want to run build using their own forks.
Also those images
+ do not provide public access, so you need to login to
``docker.pkg.github.com`` docker registry
+ using your username and personal token to be able to pull those images.
+
+* The new `GitHub Container Registry
<https://docs.github.com/en/packages/guides/about-github-container-registry>`_
+ which is in Public Beta, has many more features (including permission
management, public access and
+ image retention possibility). It has also the drawback (at least as of
January 2020) that you need to
+ have separate personal access token created as ``PAT_CR`` secret in your
repository with write access
+ to registry in order to make it works. You also have to manually manage
permissions of the images,
+ i.e. after creating images for the first time, you need to set their
visibility to "Public" and
+ add ``Admin`` permissions to group of people managing the images (in our
case ``airflow-committers`` group).
+ This makes it not very suitable to use GitHub container registry if you want
to run builds of Airflow
+ in your own forks (note - it does not affect pull requests from forks to
Airflow).
+
+Those two images have different naming schemas. See `Images documentation
<IMAGES.rst>`_ for details.
+
+You can choose which registry should be used by the repository by setting
``OVERRIDE_GITHUB_REGISTRY`` secret
+to either ``docker.pkg.github.com`` for Github Package Registry or ``ghcr.io``
for GitHub Container Registry.
+Default is the Github Package Registry one. The Pull Request forks have no
access to the secret but they
+auto-detect the registry used when they wait for the images.
+
+You can interact with the Github Registry images (pull/push) via `Breeze
<BREEZE.rst>`_ - you can
+pass ``--github-registry`` flag wih either ``docker.pkg.github.com`` for
Github Package Registry or
+``ghcr.io`` for GitHub Container Registry and pull/push operations will be
performed using the chosen
+registry, using appropriate naming convention. This allows building and
pushing the images locally by
+committers who have access to push/pull those images.
+
+
+Github Container Registry Token
+-------------------------------
+
+Unlike GitHub Packages, GitHub Registry requires a personal access token added
as ``PAT_CR`` secret in order
+to make it works. This token has to have "Registry Write" scope. Ideally you
should not use a token
+of a person who has access to many repositories, because this token allows to
write packages in
+ANY repository, where the person has write access (including private
organisations). Ideally, you need to have
+a separate account with only access to that repository and generate Personal
Access Token with Package
+Registry write permission for that Account. Discussion about setting up such
account is opened at
+`ASF Jira
<https://issues.apache.org/jira/projects/INFRA/issues/INFRA-20959>`_. More info
about
+the token for GitHub Container Registry can be found
+`here
<https://docs.github.com/en/packages/guides/migrating-to-github-container-registry-for-docker-images#authenticating-with-the-container-registry>`_
+
+
+Locally replicating CI failures
+-------------------------------
The main goal of the CI philosophy we have that no matter how complex the test
and integration
infrastructure, as a developer you should be able to reproduce and re-run any
of the failed checks
@@ -808,7 +862,7 @@ you need to reproduce a MySQL environment with kerberos
integration enabled for
.. code-block:: bash
- ./breeze --github-image-id 210056909 --python 3.8 --integration kerberos
+ ./breeze --github-image-id 210056909 --github-registry docker.pkg.github.com
--python 3.8
You will be dropped into a shell with the exact version that was used during
the CI run and you will
be able to run pytest tests manually, easily reproducing the environment that
was used in CI. Note that in
@@ -842,3 +896,51 @@ Scheduled build flow
.. image:: images/ci/scheduled_ci_flow.png
:align: center
:alt: Scheduled build flow
+
+
+Adding new Python versions to CI
+--------------------------------
+
+In 2.0 line we currently support Python 3.6, 3.7, 3.8.
+
+In order to add a new version the following operations should be done (example
uses python 3.9)
+
+* copy the latest constraints in ``constraints-master`` branch from previous
versions and name it
+ using the new Python version (``constraints-3.9.txt``). Commit and push
+
+* add the new python version to `breeze-complete <breeze-complete>`_ and
+ `_initialization.sh <scripts/ci/libraries/_initialization.sh>`_ - tests will
fail if they are not
+ in sync.
+
+* build image locally for both prod and CI locally using Breeze:
+
+.. code-block:: bash
+
+ ./breeze build-image --python 3.9
+
+* push image as cache to DockerHub and both registries:
+
+.. code-block:: bash
+
+ ./breeze push-image --python 3.9
+ ./breeze push-image --python 3.9 --github-registry ghcr.io
+ ./breeze push-image --python 3.9 --github-registry docker.pkg.github.com
+
+* Find the 3 new images (main, ci, build) created in
+ `GitHub Container
registry<https://github.com/orgs/apache/packages?tab=packages&ecosystem=container&q=airflow>`_
+ go to Package Settings and turn on ``Public Visibility`` and add
``airflow-committers``
+ group as ``Admin Role`` to all of them.
+
+* In `DockerHub
<https://hub.docker.com/repository/docker/apache/airflow/builds/edit>`_ create
three entries
+ for automatically built nightly-tag and release images:
+
+
++-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+
+| Source type | Source | Docker Tag | Dockerfile location |
Build Context | Autobuild | Build caching | Comment
|
++=============+================+=======================+=====================+===============+===========+===============+========================================================================+
+| Tag | nightly-master | master-python3.9 | Dockerfile |
/ | x | - | Nightly CI/PROD images from
successful scheduled master nightly builds |
++-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+
+| Branch | v2-0-stable | v2-0-stable-python3.9 | Dockerfile |
/ | x | | CI/PROD images automatically built
pushed stable branch |
++-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+
+| Tag | /^([1-2].*)$/ | {\1}-python3.9 | Dockerfile |
/ | x | | CI/PROD images automatically built
from pushed release tags |
++-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+
diff --git a/IMAGES.rst b/IMAGES.rst
index 102c120..c6d09b0 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -228,7 +228,12 @@ Choosing image registry
=======================
By default images are pulled and pushed from and to DockerHub registry when
you use Breeze's push-image
-or build commands.
+or build commands. But as described in `CI Documentaton <CI.rst>`_, you can
choose different image
+registry by setting ``GITHUB_REGISTRY`` to ``docker.pkg.github.com`` for
Github Package Registry or
+``ghcr.io`` for GitHub Container Registry.
+
+Default is the Github Package Registry one. The Pull Request forks have no
access to the secret but they
+auto-detect the registry used when they wait for the images.
Our images are named like that:
@@ -349,6 +354,60 @@ GitHub Container Registry
docker login ghcr.io
+Interacting with container registries
+=====================================
+
+Since there are different naming conventions used for Airflow images and there
are multiple images used,
+`Breeze <BREEZE.rst>`_ provides easy to use management interface for the
images. The
+`CI system of ours <CI.rst>`_ is designed in the way that it should
automatically refresh caches, rebuild
+the images periodically and update them whenever new version of base python is
released.
+However, occasionally, you might need to rebuild images locally and push them
directly to the registries
+to refresh them.
+
+This can be done with ``Breeze`` command line which has easy-to-use tool to
manage those images. For
+example:
+
+
+Force building Python 3.6 CI image using local cache and pushing it container
registry:
+
+.. code-block:: bash
+
+ ./breeze build-image --python 3.6 --force-build-images --build-cache-local
+ ./breeze push-image --python 3.6 --github-registry ghcr.io
+
+
+Building Python 3.7 PROD images (both build and final image) using cache pulled
+from ``docker.pkg.github.com`` and pushing it back:
+
+.. code-block:: bash
+
+ ./breeze build-image --production-image --python 3.7 --github-registry
docker.pkg.github.com
+ ./breeze push-image --production-image --python 3.7 --github-registry
docker.pkg.github.com
+
+
+Building Python 3.8 CI image using cache pulled from DockerHub and pushing it
back:
+
+.. code-block:: bash
+
+ ./breeze build-image --python 3.8
+ ./breeze push-image --python 3.8
+
+You can also pull and run images being result of a specific CI run in GitHub
Actions. This is a powerful
+tool that allows to reproduce CI failures locally, enter the images and fix
them much faster. It is enough
+to pass ``--github-image-id`` and the registry and Breeze will download and
execute commands using
+the same image that was used during the CI build.
+
+For example this command will run the same Python 3.8 image as was used in
210056909
+run with enabled Kerberos integration (assuming docker.pkg.github.com was used
as build cache).
+
+.. code-block:: bash
+
+ ./breeze --github-image-id 210056909 \
+ --github-registry docker.pkg.github.com \
+ --python 3.8 --integration kerberos
+
+You can see more details and examples in `Breeze <BREEZE.rst>`_
+
Technical details of Airflow images
===================================
diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh
b/scripts/ci/images/ci_wait_for_all_prod_images.sh
index 25bfd7c..626c3ae 100755
--- a/scripts/ci/images/ci_wait_for_all_prod_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_prod_images.sh
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
echo
echo "Waiting for all PROD images to appear:
${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
echo
diff --git a/scripts/ci/libraries/_initialization.sh
b/scripts/ci/libraries/_initialization.sh
index 832cb17..c6631d0 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -509,7 +509,7 @@ function initialization::initialize_github_variables() {
# Defaults for interacting with GitHub
export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
export GITHUB_REGISTRY_IMAGE_SUFFIX=${GITHUB_REGISTRY_IMAGE_SUFFIX:="-v2"}
- export GITHUB_REGISTRY=${GITHUB_REGISTRY:="ghcr.io"}
+ export GITHUB_REGISTRY=${GITHUB_REGISTRY:="docker.pkg.github.com"}
export
GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"}
export
GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
export
GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"}
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh
b/scripts/ci/libraries/_push_pull_remove_images.sh
index 1609e00..10c1503 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -285,74 +285,73 @@ function push_pull_remove_images::push_prod_images() {
fi
}
-# waits for an image to be available in GitHub Packages
-function push_pull_remove_images::wait_for_image_in_github_packages() {
+# waits for an image to be available in GitHub Packages. Should be run with
`set +e`
+# the buid automatically determines which registry to use based one the images
available
+function push_pull_remove_images::check_for_image_in_github_packages() {
local github_repository_lowercase
github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]'
'[:lower:]')"
local github_api_endpoint
-
github_api_endpoint="https://${GITHUB_REGISTRY}/v2/${github_repository_lowercase}";
+
github_api_endpoint="https://docker.pkg.github.com/v2/${github_repository_lowercase}";
local image_name_in_github_registry="${1}"
local image_tag_in_github_registry=${2}
-
- echo
- echo "Waiting for
${GITHUB_REPOSITORY}/${image_name_in_github_registry}:${image_tag_in_github_registry}
image"
- echo
-
-
GITHUB_API_CALL="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}"
- while true; do
- http_status=$(curl --silent --output "${OUTPUT_LOG}" --write-out
"%{http_code}" \
- --connect-timeout 60 --max-time 60 \
- -X GET "${GITHUB_API_CALL}" -u
"${GITHUB_USERNAME}:${GITHUB_TOKEN}")
- if [[ ${http_status} == "200" ]]; then
- echo "${COLOR_GREEN}OK. ${COLOR_RESET}"
- break
- else
- echo "${COLOR_YELLOW}Still waiting - status code
${http_status}!${COLOR_RESET}"
- cat "${OUTPUT_LOG}"
- fi
- sleep 60
- done
- verbosity::print_info "Found
${image_name_in_github_registry}:${image_tag_in_github_registry} image"
+ local
image_to_wait_for=${GITHUB_REPOSITORY}/${image_name_in_github_registry}:${image_tag_in_github_registry}
+ local github_api_call
+
github_api_call="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}"
+ echo "Github Packages: checking for ${image_to_wait_for} via
${github_api_call}!"
+ http_status=$(curl --silent --output "${OUTPUT_LOG}" --write-out
"%{http_code}" \
+ --connect-timeout 60 --max-time 60 \
+ -X GET "${github_api_call}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}")
+ if [[ ${http_status} == "200" ]]; then
+ echo "Image: ${image_to_wait_for} found in GitHub Packages:
${COLOR_GREEN}OK. ${COLOR_RESET}"
+ echo "::set-output name=githubRegistry::docker.pkg.github.com"
+ echo
+ echo "Setting githubRegistry output to docker.pkg.github.com"
+ echo
+ return 0
+ else
+ cat "${OUTPUT_LOG}"
+ echo "${COLOR_YELLOW}Still waiting. Status code
${http_status}!${COLOR_RESET}"
+ return 1
+ fi
}
-
-# waits for an image to be available in GitHub Container Registry
-function
push_pull_remove_images::wait_for_image_in_github_container_registry() {
+# waits for an image to be available in GitHub Container Registry. Should be
run with `set +e`
+function
push_pull_remove_images::check_for_image_in_github_container_registry() {
local image_name_in_github_registry="${1}"
local image_tag_in_github_registry=${2}
- local
image_to_wait_for="${GITHUB_REGISTRY}/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry}"
- echo
- echo "Waiting for
${GITHUB_REGISTRY}/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry}
image"
- echo
+ local
image_to_wait_for="ghcr.io/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry}"
+ echo "Github Container Registry: checking for ${image_to_wait_for} via
docker manifest inspect!"
+ docker manifest inspect "${image_to_wait_for}"
+ local res=$?
+ if [[ ${res} == "0" ]]; then
+ echo "Image: ${image_to_wait_for} found in Container Registry:
${COLOR_GREEN}OK.${COLOR_RESET}"
+ echo
+ echo "Setting githubRegistry output to ghcr.io"
+ echo
+ echo "::set-output name=githubRegistry::ghcr.io"
+ return 0
+ else
+ echo "${COLOR_YELLOW}Still waiting. Not found!${COLOR_RESET}"
+ return 1
+ fi
+}
+
+# waits for an image to be available in the GitHub registry
+function push_pull_remove_images::wait_for_github_registry_image() {
set +e
- while true; do
- docker manifest inspect "${image_to_wait_for}"
- local res=$?
- if [[ ${res} == "0" ]]; then
- echo "${COLOR_GREEN}OK.${COLOR_RESET}"
+ echo " Waiting for github registry image: " "${@}"
+ while true
+ do
+ if
push_pull_remove_images::check_for_image_in_github_container_registry "${@}";
then
+ break
+ fi
+ if push_pull_remove_images::check_for_image_in_github_packages "${@}";
then
break
- else
- echo "${COLOR_YELLOW}Still waiting for
${image_to_wait_for}!${COLOR_RESET}"
fi
sleep 30
done
set -e
- verbosity::print_info "Found
${image_name_in_github_registry}:${image_tag_in_github_registry} image"
-}
-
-# waits for an image to be available in the GitHub registry
-function push_pull_remove_images::wait_for_github_registry_image() {
- if [[ ${GITHUB_REGISTRY} == "ghcr.io" ]]; then
- push_pull_remove_images::wait_for_image_in_github_container_registry
"${@}"
- elif [[ ${GITHUB_REGISTRY} == "docker.pkg.github.com" ]]; then
- push_pull_remove_images::wait_for_image_in_github_packages "${@}"
- else
- echo
- echo "${COLOR_RED}ERROR: Bad value of '${GITHUB_REGISTRY}'. Should be
either 'ghcr.io' or 'docker.pkg.github.com'!${COLOR_RESET}"
- echo
- exit 1
- fi
}
function
push_pull_remove_images::check_if_github_registry_wait_for_image_enabled() {
