This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-1-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 03a23e015122641f7a6c459ac20505e91321d1df Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Jul 4 13:44:51 2021 +0200 Remove legacy GitHub Packages (#16776) This PR removes the legacy GitHub Packages support: * removes checking for images in Packages/Registry * removes output informing about the registry * hard-codes registry to ghcr.io * Updaes documentation describing the registries (cherry picked from commit b251d22fffad63124eec5246b80035408b543704) --- .github/workflows/build-images.yml | 1 - .github/workflows/ci.yml | 33 +------ BREEZE.rst | 30 ------ CI.rst | 44 +++------ IMAGES.rst | 113 +++++------------------ breeze | 69 +++++--------- breeze-complete | 6 -- scripts/ci/libraries/_initialization.sh | 3 +- scripts/ci/libraries/_push_pull_remove_images.sh | 36 -------- 9 files changed, 65 insertions(+), 270 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 9097241..9cbae1d 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -47,7 +47,6 @@ env: GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" INSTALL_PROVIDERS_FROM_SOURCES: "true" - GITHUB_REGISTRY: "ghcr.io" TARGET_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} concurrency: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57ab690..6763886 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,6 @@ env: VERBOSE: "true" DOCKER_CACHE: "pulled" USE_GITHUB_REGISTRY: "true" - GITHUB_REGISTRY: "ghcr.io" GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_USERNAME: ${{ github.actor }} # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the @@ -266,8 +265,6 @@ jobs: BACKEND: sqlite UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }} - outputs: - githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -295,9 +292,6 @@ jobs: # We are utilising single job to wait for all images because this job merely waits # for the images to be available. # The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'! - # The job will set the output "githubRegistry" - result of auto-detect which registry has - # been used by checking where the image can be downloaded from. - # run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh @@ -311,7 +305,6 @@ jobs: SKIP: "identity" MOUNT_SELECTED_LOCAL_SOURCES: "true" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: needs.build-info.outputs.basic-checks-only == 'false' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -417,7 +410,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" if: needs.build-info.outputs.docs-build == 'true' env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -465,7 +457,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" AIRFLOW_EXTRAS: "all" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} VERSION_SUFFIX_FOR_PYPI: ".dev0" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} NON_INTERACTIVE: "true" GENERATE_PROVIDERS_ISSUE: "true" if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' @@ -514,7 +505,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" AIRFLOW_EXTRAS: "all" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} VERSION_SUFFIX_FOR_PYPI: ".dev0" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} NON_INTERACTIVE: "true" GENERATE_PROVIDERS_ISSUE: "true" if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' @@ -558,7 +548,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" BACKEND: "" DB_RESET: "false" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: > needs.build-info.outputs.needs-helm-tests == 'true' && (github.repository == 'apache/airflow' || github.event_name != 'schedule') @@ -618,7 +607,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} POSTGRES_VERSION: ${{ matrix.postgres-version }} TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -675,7 +663,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} MYSQL_VERSION: ${{ matrix.mysql-version }} TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -729,7 +716,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -781,7 +767,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" TEST_TYPES: "Quarantined" NUM_RUNS: 10 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -880,8 +865,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - outputs: - githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -904,8 +887,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" # "build-images-workflow-run.yml' run as pull_request_target. # We are utilising single job to wait for all images because this job merely waits # For the images to be available. The test jobs wait for it to complete! - # The job will set the output "githubRegistry" - result of auto-detect which registry has - # been used by checking where the image can be downloaded from. # id: wait-for-images env: @@ -930,12 +911,13 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" EXECUTOR: ${{matrix.executor}} KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" - GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > ${{needs.build-info.outputs.pythonVersionsListAsString}} CURRENT_KUBERNETES_VERSIONS_AS_STRING: > ${{needs.build-info.outputs.kubernetesVersionsListAsString}} - if: needs.build-info.outputs.run-kubernetes-tests == 'true' + if: > + needs.build-info.outputs.run-kubernetes-tests == 'true' || + needs.build-info.outputs.needs-helm-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -994,7 +976,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" EXECUTOR: "KubernetesExecutor" KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" - GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > ${{needs.build-info.outputs.pythonVersionsListAsString}} CURRENT_KUBERNETES_VERSIONS_AS_STRING: > @@ -1058,7 +1039,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - tests-kubernetes - prod-images - docs - # TODO: Generalize me (find a better way to select matching branches) if: > (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v1-10-test' || github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test') && @@ -1070,7 +1050,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -1120,7 +1099,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - tests-kubernetes - ci-images - docs - # TODO: Generalize me (find a better way to select matching branches) if: > (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v1-10-test' || github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test') && @@ -1132,7 +1110,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -1167,10 +1144,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}} # Only run it for direct pushes - # TODO: Generalize me (find a better way to select matching branches) if: > github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v1-10-test' || github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test' @@ -1264,8 +1239,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" name: React UI tests runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, ci-images] - env: - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: needs.build-info.outputs.run-ui-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" diff --git a/BREEZE.rst b/BREEZE.rst index d199367..0bb83fe 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1445,16 +1445,6 @@ This is the current syntax for `./breeze <./breeze>`_: DockerHub. You need to be logged in to the registry in order to be able to pull/push from and you need to be committer to push to Apache Airflow' GitHub registry. - --github-registry GITHUB_REGISTRY - GitHub registry used. GitHub has legacy Packages registry and Public Beta Container - registry. - - Default: ghcr.io. - - If you use this flag, automatically --use-github-registry flag is enabled. - - ghcr.io docker.pkg.github.com - -g, --github-repository GITHUB_REPOSITORY GitHub repository used to pull, push images when cache is used. Default: apache/airflow. @@ -1619,16 +1609,6 @@ This is the current syntax for `./breeze <./breeze>`_: DockerHub. You need to be logged in to the registry in order to be able to pull/push from and you need to be committer to push to Apache Airflow' GitHub registry. - --github-registry GITHUB_REGISTRY - GitHub registry used. GitHub has legacy Packages registry and Public Beta Container - registry. - - Default: ghcr.io. - - If you use this flag, automatically --use-github-registry flag is enabled. - - ghcr.io docker.pkg.github.com - -g, --github-repository GITHUB_REPOSITORY GitHub repository used to pull, push images when cache is used. Default: apache/airflow. @@ -2692,16 +2672,6 @@ This is the current syntax for `./breeze <./breeze>`_: DockerHub. You need to be logged in to the registry in order to be able to pull/push from and you need to be committer to push to Apache Airflow' GitHub registry. - --github-registry GITHUB_REGISTRY - GitHub registry used. GitHub has legacy Packages registry and Public Beta Container - registry. - - Default: ghcr.io. - - If you use this flag, automatically --use-github-registry flag is enabled. - - ghcr.io docker.pkg.github.com - -g, --github-repository GITHUB_REPOSITORY GitHub repository used to pull, push images when cache is used. Default: apache/airflow. diff --git a/CI.rst b/CI.rst index 4b92d2d..dcc859d 100644 --- a/CI.rst +++ b/CI.rst @@ -54,7 +54,7 @@ it can be ~6-7 minutes and in case base image of Python releases new patch-level Container Registry used as cache -------------------------------- -For the CI builds of our we are using Container Registry to store results of the "Build Image" workflow +For the CI builds of our we are using GitHub Container Registry to store results of the "Build Image" workflow and pass it to the "CI Build" workflow. Currently in main version of Airflow we run tests in 4 different versions of Python (3.6, 3.7, 3.8, 3.9) @@ -70,40 +70,23 @@ This is especially important in our case where Pull Requests to Airflow might co and it would be a huge security issue if anyone from outside could utilise the WRITE access to Apache Airflow repository via an external Pull Request. -Thanks to the WRITE access and fact that the 'pull_request_target' by default uses the 'main' version of the +Thanks to the WRITE access and fact that the ``pull_request_target`` by default uses the ``main`` version of the sources, we can safely run some logic there will checkout the incoming Pull Request, build the container image from the sources from the incoming PR and push such image to an GitHub Docker Registry - so that this image can be built only once and used by all the jobs running tests. The image is tagged with unique ``COMMIT_SHA`` of the incoming Pull Request and the tests run in the Pull Request can simply pull such image rather than build it from the scratch. Pulling such image takes ~ 1 minute, thanks to that we are saving a lot of precious time for jobs. +4 +We use `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_ +GitHub Package Registry ``GITHUB_TOKEN`` is needed to push to the registry. You also have to manually manage +permissions of the images, after creating image for the first time (pushing it using your personal token) +you need to set their visibility to ``Public`` and enable +`Inheriting access from repository <https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility#inheriting-access-for-a-container-image-from-a-repository>`_ +Those images have specific naming schema. See `Images documentation <IMAGES.rst>`_ for details. -We can use either of the two available GitHub Container registries as cache: - -* Legacy `GitHub Package Registry <https://github.com/features/packages>`_ which is not very - stable, uses old infrastructure of GitHub and it lacks certain features - notably it does not allow - us to delete the old image. The benefit of using GitHub Package Registry is that it works - out-of-the-box (write authentication is done using ``GITHUB_TOKEN`` and users do not have to do any - action to make it work in case they want to run build using their own forks. Also those images - do not provide public access, so you need to login to ``docker.pkg.github.com`` docker registry - using your username and personal token to be able to pull those images. - -* The new `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_ - which is in Public Beta, has many more features (including permission management, public access and - image retention possibility). Similarly as in case of GitHub Package Registry ``GITHUB_TOKEN`` is needed - to push to the repositories. You also have to manually manage permissions of the images, - i.e. after creating images for the first time, you need to set their visibility to ``Public`` and - add ``Admin`` permissions to group of people managing the images (in our case ``airflow-committers`` group). - This makes it not very suitable to use GitHub container registry if you want to run builds of Airflow - in your own forks (note - it does not affect pull requests from forks to Airflow). - -Those two images have different naming schemas. See `Images documentation <IMAGES.rst>`_ for details. - -You can interact with the GitHub Registry images (pull/push) via `Breeze <BREEZE.rst>`_ - you can -pass ``--github-registry`` flag with either ``docker.pkg.github.com`` for GitHub Package Registry or -``ghcr.io`` for GitHub Container Registry and pull/push operations will be performed using the chosen -registry, using appropriate naming convention. This allows building and pushing the images locally by -committers who have access to push/pull those images. +You can interact with the GitHub Registry images (pull/push) via `Breeze <BREEZE.rst>`_ - by passing +``--use-github-registry`` flag. Locally replicating CI failures ------------------------------- @@ -787,7 +770,7 @@ cd27124534b46c9688a1d89e75fcd137ab5137e3, in python 3.8 environment you can run: .. code-block:: bash - ./breeze --github-image-id cd27124534b46c9688a1d89e75fcd137ab5137e3 --github-registry ghcr.io --python 3.8 + ./breeze --github-image-id cd27124534b46c9688a1d89e75fcd137ab5137e3 --use=github-registry --python 3.8 You will be dropped into a shell with the exact version that was used during the CI run and you will be able to run pytest tests manually, easily reproducing the environment that was used in CI. Note that in @@ -848,8 +831,7 @@ In order to add a new version the following operations should be done (example u .. code-block:: bash ./breeze push-image --python 3.9 - ./breeze push-image --python 3.9 --github-registry ghcr.io - ./breeze push-image --python 3.9 --github-registry docker.pkg.github.com + ./breeze push-image --python 3.9 --use-github-registry * Find the 3 new images (main, ci, build) created in `GitHub Container registry <https://github.com/orgs/apache/packages?tab=packages&ecosystem=container&q=airflow>`_ diff --git a/IMAGES.rst b/IMAGES.rst index c3a1805..47ce39f 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -270,14 +270,11 @@ Choosing image registry ======================= By default images are pulled and pushed from and to DockerHub registry when you use Breeze's push-image -or build commands. But as described in `CI Documentation <CI.rst>`_, you can choose different image -registry by setting ``GITHUB_REGISTRY`` to ``docker.pkg.github.com`` for GitHub Package Registry or -``ghcr.io`` for GitHub Container Registry. +or build commands. -Default is the GitHub Package Registry one. The Pull Request forks have no access to the secret but they -auto-detect the registry used when they wait for the images. +But as described in `CI Documentation <CI.rst>`_, you can choose GitHub Container Registry. -Our images are named following conventions below. +Naming convention for DockerHub images. Images used during CI builds: @@ -320,53 +317,16 @@ For example: You can see those CI DockerHub images at `<https://hub.docker.com/r/apache/airflow>`_ -Using GitHub registries as build cache --------------------------------------- +Using GitHub Container Registry as build cache +---------------------------------------------- -By default DockerHub registry is used when you push or pull such images. -However for CI builds we keep the images in GitHub registry as well - this way we can easily push -the images automatically after merge requests and use such images for Pull Requests -as cache - which makes it much it much faster for CI builds (images are available in cache -right after merged request in main finishes it's build), The difference is visible especially if -significant changes are done in the Dockerfile.CI. - -The images are named differently (in Docker definition of image names - registry URL is part of the -image name if DockerHub is not used as registry). Also GitHub has its own structure for registries -each project has its own registry naming convention that should be followed. The name of -images for GitHub registry are different as they must follow limitation of the registry used. - -We are still using GitHub Packages as registry, but we are in the process of testing and switching -to GitHub Container Registry, and the naming conventions are slightly different (GitHub Packages -required all packages to have "organization/repository/" URL prefix ("apache/airflow/", -where in GitHub Container Registry, all images are in "organization" not in "repository" and they are all -in organization wide "apache/" namespace rather than in "apache/airflow/" one). -We are adding "airflow-" as prefix for image names of all Airflow images instead. +We are using GitHub Container Registry as build cache.The images are all in organization wide "apache/" +namespace. We are adding "airflow-" as prefix for image names of all Airflow images. The images are linked to the repository via ``org.opencontainers.image.source`` label in the image. -Naming convention for GitHub Packages -------------------------------------- - -Images with a commit SHA (built for pull requests and pushes) - -.. code-block:: bash - - docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-ci-v2:<COMMIT_SHA> - for CI images - docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-v2:<COMMIT_SHA> - for production images - docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-build-v2:<COMMIT_SHA> - for production build stage - docker.pkg.github.com/apache-airflow/python-v2:X.Y-slim-buster-<COMMIT_SHA> - for base Python images - -Latest images (pushed when main merge succeeds): - -.. code-block:: bash - - docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-ci-v2:latest - for CI images - docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-v2:latest - for production images - docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-build-v2:latest - for production build stage - docker.pkg.github.com/apache/airflow/python-v2:X.Y-slim-buster - for base Python images - +See https://docs.github.com/en/packages/learn-github-packages/connecting-a-repository-to-a-package -Naming convention for GitHub Container Registry ------------------------------------------------ +Naming convention for the GitHub packages. Images with a commit SHA (built for pull requests and pushes) @@ -386,40 +346,28 @@ Latest images (pushed when main merge succeeds): ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-build-v2:latest - for production build stage ghcr.io/apache/airflow-python-v2:X.Y-slim-buster - for base Python images -Note that we never push or pull "release" images to GitHub registry. It is only used for CI builds - +Note that we never push or pull "release" images to GitHub registry. Those are only pushed to DockerHub. You can see all the current GitHub images at `<https://github.com/apache/airflow/packages>`_ - -In order to interact with the GitHub images you need to add ``--use-github-registry`` flag to the pull/push -commands in Breeze. This way the images will be pulled/pushed from/to GitHub rather than from/to -DockerHub. Images are build locally as ``apache/airflow`` images but then they are tagged with the right -GitHub tags for you. You can also specify ``--github-registry`` option and choose which of the -GitHub registries are used (``docker.pkg.github.com`` chooses GitHub Packages and ``ghcr.io`` chooses -GitHub Container Registry). +In order to interact with the GitHub Container Registry you need to add ``--use-github-registry`` +flag to the pull/push commands in Breeze. This way the images will be pulled/pushed from/to GitHub +rather than from/to DockerHub. Images are build locally as ``apache/airflow`` images but then they are +tagged with the right GitHub tags for you automatically. You can read more about the CI configuration and how CI builds are using DockerHub/GitHub images in `<CI.rst>`_. Note that you need to be committer and have the right to push to DockerHub and GitHub and you need to -be logged in. Only committers can push images directly. You need to login with your -Personal Access Token with "packages" scope to be able to push to those repositories or pull from them +be logged in to the registry. Only committers can push images directly. You need to login with your +Personal Access Token with "packages" write scope to be able to push to those repositories or pull from them in case of GitHub Packages. -GitHub Packages: - -.. code-block:: bash - - docker login docker.pkg.github.com - GitHub Container Registry .. code-block:: bash docker login ghcr.io -Interacting with container registries -===================================== Since there are different naming conventions used for Airflow images and there are multiple images used, `Breeze <BREEZE.rst>`_ provides easy to use management interface for the images. The @@ -431,23 +379,12 @@ to refresh them. This can be done with ``Breeze`` command line which has easy-to-use tool to manage those images. For example: - Force building Python 3.6 CI image using local cache and pushing it container registry: .. code-block:: bash - ./breeze build-image --python 3.6 --force-build-images --build-cache-local - ./breeze push-image --python 3.6 --github-registry ghcr.io - - -Building Python 3.7 PROD images (both build and final image) using cache pulled -from ``docker.pkg.github.com`` and pushing it back: - -.. code-block:: bash - - ./breeze build-image --production-image --python 3.7 --github-registry docker.pkg.github.com - ./breeze push-image --production-image --python 3.7 --github-registry docker.pkg.github.com - + ./breeze build-image --python 3.6 --force-build-images --force-pull-base-python-image --build-cache-local + ./breeze push-image --python 3.6 --use-github-registry Building Python 3.8 CI image using cache pulled from DockerHub and pushing it back: @@ -461,13 +398,13 @@ tool that allows to reproduce CI failures locally, enter the images and fix them to pass ``--github-image-id`` and the registry and Breeze will download and execute commands using the same image that was used during the CI build. -For example this command will run the same Python 3.8 image as was used in 210056909 -run with enabled Kerberos integration (assuming docker.pkg.github.com was used as build cache). +For example this command will run the same Python 3.8 image as was used in build identified with +9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e commit SHA with enabled Kerberos integration. .. code-block:: bash - ./breeze --github-image-id 210056909 \ - --github-registry docker.pkg.github.com \ + ./breeze --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e \ + --use-github-registry \ --python 3.8 --integration kerberos You can see more details and examples in `Breeze <BREEZE.rst>`_ @@ -774,11 +711,9 @@ GitHub Registies in order to be able to do that. ./breeze build-image --python ${python_version} --build-cache-local \ --production-image --verbose ./breeze push-image - ./breeze push-image --github-registry ghcr.io - ./breeze push-image --github-registry docker.pkg.github.com + ./breeze push-image --use-github-registry ./breeze push-image --production-image - ./breeze push-image --github-registry ghcr.io --production-image - ./breeze push-image --github-registry docker.pkg.github.com --production-image + ./breeze push-image --production-image --use-github-registry done Running the CI image diff --git a/breeze b/breeze index 9a352fd..6240d07 100755 --- a/breeze +++ b/breeze @@ -572,6 +572,7 @@ EOF # BACKEND # AIRFLOW_VERSION # INSTALL_AIRFLOW_VERSION +# SSH_PORT # WEBSERVER_HOST_PORT # POSTGRES_HOST_PORT # POSTGRES_VERSION @@ -620,6 +621,7 @@ export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" export BACKEND="${BACKEND}" export AIRFLOW_VERSION="${AIRFLOW_VERSION}" export INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}" +export SSH_PORT="${SSH_PORT}" export WEBSERVER_HOST_PORT="${WEBSERVER_HOST_PORT}" export FLOWER_HOST_PORT="${FLOWER_HOST_PORT}" export REDIS_HOST_PORT="${REDIS_HOST_PORT}" @@ -1141,15 +1143,6 @@ function breeze::parse_arguments() { export USE_GITHUB_REGISTRY="true" shift ;; - --github-registry) - echo - echo "Using GitHub registry." - echo "GitHub registry used: ${2}" - echo - export GITHUB_REGISTRY="${2}" - export USE_GITHUB_REGISTRY="true" - shift 2 - ;; -g | --github-repository) echo echo "Using GitHub registry." @@ -1536,6 +1529,12 @@ function breeze::parse_arguments() { # ####################################################################################################### function breeze::prepare_formatted_versions() { + if [[ -n "${_breeze_formatted_versions_prepared:-}" ]]; then + return + fi + + _breeze_formatted_versions_prepared=1 + local indent=15 local list_prefix list_prefix=$(printf "%-${indent}s" " ") @@ -1593,10 +1592,6 @@ function breeze::prepare_formatted_versions() { tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") readonly FORMATTED_GENERATE_CONSTRAINTS_MODE - FORMATTED_GITHUB_REGISTRY=$(echo "${_breeze_allowed_github_registrys=""}" | - tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") - readonly FORMATTED_GITHUB_REGISTRY - FORMATTED_POSTGRES_VERSIONS=$(echo "${_breeze_allowed_postgres_versions=""}" | tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") readonly FORMATTED_POSTGRES_VERSIONS @@ -1641,6 +1636,14 @@ function breeze::prepare_formatted_versions() { # shellcheck disable=SC2034,SC2090,SC2089,SC2155 function breeze::prepare_usage() { + if [[ -n "${_breeze_usage_prepared:-}" ]]; then + return + fi + + _breeze_usage_prepared=1 + + breeze::prepare_formatted_versions + # Note that MacOS uses Bash 3.* and we cannot use associative arrays export USAGE_SHELL="[Default] Enters interactive shell in the container" readonly USAGE_SHELL @@ -2133,6 +2136,7 @@ function breeze::get_variable_from_lowercase_name() { # usage information for the command. ####################################################################################################### function breeze::get_usage() { + breeze::prepare_usage breeze::get_variable_from_lowercase_name "USAGE" "${1}" } @@ -2145,6 +2149,7 @@ function breeze::get_usage() { # Detailed usage information for the command. ####################################################################################################### function breeze::get_detailed_usage() { + breeze::prepare_usage breeze::get_variable_from_lowercase_name "DETAILED_USAGE" "${1}" } @@ -2162,6 +2167,7 @@ function breeze::get_detailed_usage() { # General usage information for all commands. ####################################################################################################### function breeze::usage() { + breeze::prepare_usage echo " usage: ${CMDNAME} [FLAGS] [COMMAND] -- <EXTRA_ARGS> @@ -2804,16 +2810,6 @@ function breeze::flag_pull_push_docker_images() { DockerHub. You need to be logged in to the registry in order to be able to pull/push from and you need to be committer to push to Apache Airflow' GitHub registry. ---github-registry GITHUB_REGISTRY - GitHub registry used. GitHub has legacy Packages registry and Public Beta Container - registry. - - Default: ${_breeze_default_github_registry:=}. - - If you use this flag, automatically --use-github-registry flag is enabled. - -${FORMATTED_GITHUB_REGISTRY} - -g, --github-repository GITHUB_REPOSITORY GitHub repository used to pull, push images when cache is used. Default: ${_breeze_default_github_repository:=}. @@ -2919,6 +2915,8 @@ ${FORMATTED_TEST_TYPES} # Flag information. ####################################################################################################### function breeze::flags() { + breeze::prepare_formatted_versions + echo " $(breeze::print_line) @@ -3082,18 +3080,6 @@ function breeze::read_saved_environment_variables() { MYSQL_VERSION="${MYSQL_VERSION:=$(parameters::read_from_file MYSQL_VERSION)}" MYSQL_VERSION=${MYSQL_VERSION:=${_breeze_default_mysql_version}} - - # Here you read DockerHub user/account that you use - # You can populate your own images in DockerHub this way and work with the, - # You can override it with "--dockerhub-user" option and it will be stored in .build directory - DOCKERHUB_USER="${DOCKERHUB_USER:=$(parameters::read_from_file DOCKERHUB_USER)}" - DOCKERHUB_USER="${DOCKERHUB_USER:=${_breeze_default_dockerhub_user}}" - - # Here you read DockerHub repo that you use - # You can populate your own images in DockerHub this way and work with them - # You can override it with "--dockerhub-repo" option and it will be stored in .build directory - DOCKERHUB_REPO="${DOCKERHUB_REPO:=$(parameters::read_from_file DOCKERHUB_REPO)}" - DOCKERHUB_REPO="${DOCKERHUB_REPO:=${_breeze_default_dockerhub_repo}}" } ####################################################################################################### @@ -3149,15 +3135,9 @@ function breeze::check_and_save_all_params() { parameters::check_and_save_allowed_param "EXECUTOR" "Executors" "--executor" parameters::check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version" parameters::check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version" - parameters::check_and_save_allowed_param "GITHUB_REGISTRY" "GitHub Registry" "--github-registry" parameters::check_allowed_param TEST_TYPE "Type of tests" "--test-type" parameters::check_allowed_param PACKAGE_FORMAT "Format of packages to build" "--package-format" - - - # Can't verify those - they can be anything, so let's just save them - parameters::save_to_file DOCKERHUB_USER - parameters::save_to_file DOCKERHUB_REPO } ####################################################################################################### @@ -3167,6 +3147,7 @@ function breeze::check_and_save_all_params() { # Used global constants: # # AIRFLOW_SOURCES +# SSH_PORT # WEBSERVER_HOST_PORT # POSTGRES_HOST_PORT # MYSQL_HOST_PORT @@ -3198,6 +3179,7 @@ function breeze::print_cheatsheet() { echo " Port forwarding:" echo echo " Ports are forwarded to the running docker containers for webserver and database" + echo " * ${SSH_PORT} -> forwarded to Airflow ssh server -> airflow:22" echo " * ${WEBSERVER_HOST_PORT} -> forwarded to Airflow webserver -> airflow:8080" echo " * ${FLOWER_HOST_PORT} -> forwarded to Flower dashboard -> airflow:5555" echo " * ${POSTGRES_HOST_PORT} -> forwarded to Postgres database -> postgres:5432" @@ -3205,6 +3187,7 @@ function breeze::print_cheatsheet() { echo " * ${REDIS_HOST_PORT} -> forwarded to Redis broker -> redis:6379" echo echo " Here are links to those services that you can use on host:" + echo " * ssh connection for remote debugging: ssh -p ${SSH_PORT} [email protected] pw: airflow" echo " * Webserver: http://127.0.0.1:${WEBSERVER_HOST_PORT}"; echo " * Flower: http://127.0.0.1:${FLOWER_HOST_PORT}"; echo " * Postgres: jdbc:postgresql://127.0.0.1:${POSTGRES_HOST_PORT}/airflow?user=postgres&password=airflow" @@ -3653,10 +3636,6 @@ start_end::script_start traps::add_trap start_end::script_end EXIT -breeze::prepare_formatted_versions - -breeze::prepare_usage - set +u breeze::parse_arguments "${@}" diff --git a/breeze-complete b/breeze-complete index 65a3ee0..58af21b 100644 --- a/breeze-complete +++ b/breeze-complete @@ -27,8 +27,6 @@ _breeze_allowed_python_major_minor_versions="3.6 3.7 3.8 3.9" _breeze_allowed_backends="sqlite mysql postgres" _breeze_allowed_integrations="cassandra kerberos mongo openldap pinot rabbitmq redis statsd trino all" _breeze_allowed_generate_constraints_modes="source-providers pypi-providers no-providers" -# registrys is good here even if it is not correct english. We are adding s automatically to all variables -_breeze_allowed_github_registrys="ghcr.io docker.pkg.github.com" _breeze_allowed_kubernetes_modes="image" _breeze_allowed_kubernetes_versions="v1.20.2 v1.19.7 v1.18.15" _breeze_allowed_helm_versions="v3.2.4" @@ -45,7 +43,6 @@ _breeze_allowed_installation_methods=". apache-airflow" { # Default values for the commands & flags used _breeze_default_backend=$(echo "${_breeze_allowed_backends}" | awk '{print $1}') - _breeze_default_github_registry=$(echo "${_breeze_allowed_github_registrys}" | awk '{print $1}') _breeze_default_generate_providers_mode=$(echo "${_breeze_allowed_generate_constraints_modes}" | awk '{print $1}') _breeze_default_kubernetes_mode=$(echo "${_breeze_allowed_kubernetes_modes}" | awk '{print $1}') _breeze_default_kubernetes_version=$(echo "${_breeze_allowed_kubernetes_versions}" | awk '{print $1}') @@ -314,9 +311,6 @@ function breeze_complete::get_known_values_breeze() { --installation-method) _breeze_known_values="${_breeze_allowed_installation_methods}" ;; - --github-registry) - _breeze_known_values="${_breeze_allowed_github_registrys}" - ;; --generate-constraints-mode) _breeze_known_values="${_breeze_allowed_generate_constraints_modes}" ;; diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 2b5458e..9895d8d 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -544,9 +544,9 @@ function initialization::initialize_git_variables() { function initialization::initialize_github_variables() { # Defaults for interacting with GitHub + export GITHUB_REGISTRY="ghcr.io" export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"} export GITHUB_REGISTRY_IMAGE_SUFFIX=${GITHUB_REGISTRY_IMAGE_SUFFIX:="-v2"} - export GITHUB_REGISTRY=${GITHUB_REGISTRY:="ghcr.io"} export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"} export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"} export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"} @@ -708,7 +708,6 @@ Production image build variables: Detected GitHub environment: USE_GITHUB_REGISTRY: '${USE_GITHUB_REGISTRY}' - GITHUB_REGISTRY: '${GITHUB_REGISTRY}' GITHUB_REPOSITORY: '${GITHUB_REPOSITORY}' GITHUB_USERNAME: '${GITHUB_USERNAME}' GITHUB_TOKEN: '${GITHUB_TOKEN}' diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index a6a1c5c..f8a1d3e 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -291,35 +291,6 @@ function push_pull_remove_images::push_prod_images() { fi } -# waits for an image to be available in GitHub Packages. Should be run with `set +e` -# the build automatically determines which registry to use based one the images available -function push_pull_remove_images::check_for_image_in_github_packages() { - local github_repository_lowercase - github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')" - local github_api_endpoint - github_api_endpoint="https://docker.pkg.github.com/v2/${github_repository_lowercase}"; - local image_name_in_github_registry="${1}" - local image_tag_in_github_registry=${2} - local image_to_wait_for=${GITHUB_REPOSITORY}/${image_name_in_github_registry}:${image_tag_in_github_registry} - local github_api_call - github_api_call="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}" - echo "GitHub Packages: checking for ${image_to_wait_for} via ${github_api_call}!" - http_status=$(curl --silent --output "${OUTPUT_LOG}" --write-out "%{http_code}" \ - --connect-timeout 60 --max-time 60 \ - -X GET "${github_api_call}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}") - if [[ ${http_status} == "200" ]]; then - echo "Image: ${image_to_wait_for} found in GitHub Packages: ${COLOR_GREEN}OK. ${COLOR_RESET}" - echo "::set-output name=githubRegistry::docker.pkg.github.com" - echo - echo "Setting githubRegistry output to docker.pkg.github.com" - echo - return 0 - else - cat "${OUTPUT_LOG}" - echo "${COLOR_YELLOW}Still waiting. Status code ${http_status}!${COLOR_RESET}" - return 1 - fi -} # waits for an image to be available in GitHub Container Registry. Should be run with `set +e` function push_pull_remove_images::check_for_image_in_github_container_registry() { @@ -332,10 +303,6 @@ function push_pull_remove_images::check_for_image_in_github_container_registry() local res=$? if [[ ${res} == "0" ]]; then echo "Image: ${image_to_wait_for} found in Container Registry: ${COLOR_GREEN}OK.${COLOR_RESET}" - echo - echo "Setting githubRegistry output to ghcr.io" - echo - echo "::set-output name=githubRegistry::ghcr.io" return 0 else echo "${COLOR_YELLOW}Still waiting. Not found!${COLOR_RESET}" @@ -352,9 +319,6 @@ function push_pull_remove_images::wait_for_github_registry_image() { if push_pull_remove_images::check_for_image_in_github_container_registry "${@}"; then break fi - if push_pull_remove_images::check_for_image_in_github_packages "${@}"; then - break - fi sleep 30 done set -e
