This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new a6216a7 You can push with Breeze as separate command and to cache
(#8976)
a6216a7 is described below
commit a6216a760ca7449cfbf5aeee0f74caeda2f3f999
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Jun 2 10:24:47 2020 +0200
You can push with Breeze as separate command and to cache (#8976)
Breeze had --push-images switch to also push images to repo
but it was often needed to build and push images separately.
We have now a possibility to push an already built image with
separate push-image command instead and also you can choose
to push to cache registry in GitHub rather than to DockerHub
with --registry-cache switch.
---
.github/workflows/ci.yml | 2 +-
BREEZE.rst | 84 +++++++++++++++++---
breeze | 102 +++++++++++++++++++------
breeze-complete | 13 +++-
scripts/ci/_utils.sh | 35 +++++----
scripts/ci/ci_fix_ownership.sh | 9 +++
scripts/ci/in_container/_in_container_utils.sh | 2 +-
7 files changed, 194 insertions(+), 53 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f751df6..aedfc46 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -34,7 +34,7 @@ env:
VERBOSE: "true"
UPGRADE_TO_LATEST_REQUIREMENTS: "false"
PYTHON_MAJOR_MINOR_VERSION: 3.6
- CACHE_REGISTRY: "docker.pkg.github.com"
+ ENABLE_REGISTRY_CACHE: "true"
CACHE_IMAGE_PREFIX: ${{ github.repository }}
CACHE_REGISTRY_USERNAME: ${{ github.actor }}
CACHE_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
diff --git a/BREEZE.rst b/BREEZE.rst
index d2e27c3..c05cd9d 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -669,6 +669,7 @@ This is the current syntax for `./breeze <./breeze>`_:
generate-requirements Generates pinned requirements for
pip dependencies
prepare-backport-readme Prepares backport packages readme
files
prepare-backport-packages Prepares backport packages
+ push-image Pushes images to registry
initialize-local-virtualenv Initializes local virtualenv
setup-autocomplete Sets up autocomplete for breeze
stop Stops the docker-compose
environment
@@ -802,17 +803,23 @@ This is the current syntax for `./breeze <./breeze>`_:
Uses local cache to build images. No pulled images will be used, but
results of local
builds in the Docker cache are used instead.
- -u, --push-images
- After building - uploads the images to DockerHub
- It is useful in case you use your own DockerHub user to store images
and you want
- to build them locally. Note that you need to use 'docker login'
before you upload images.
-
-D, --dockerhub-user
DockerHub user used to pull, push and build images. Default: apache.
-H, --dockerhub-repo
DockerHub repository used to pull, push, build images. Default:
airflow.
+ -c, --registry-cache
+ If registry cache is enabled, pulls and pushes are done from the
registry cache in github.
+ You need to be logged in to the registry in order to be able to
pull/push from it and you
+ need to be committer to push to airflow registry.
+
+ -G, --github-organisation
+ GitHub organisation used to pull, push images when cache is used.
Default: apache.
+
+ -g, --github-repo
+ GitHub repository used to pull, push images when cache is used.
Default: airflow.
+
-v, --verbose
Show verbose information about executed commands (enabled by default
for running test).
Note that you can further increase verbosity and see all the
commands executed by breeze
@@ -973,6 +980,55 @@ This is the current syntax for `./breeze <./breeze>`_:
####################################################################################################
+ Detailed usage for command: push-image
+
+ breeze [FLAGS] push-image -- <EXTRA_ARGS>
+
+ Pushes images to docker registry. You can push the images to DockerHub
registry (default)
+ or to the GitHub cache registry (if --registry-cache flag is used).
+
+ For DockerHub pushes --dockerhub-user and --dockerhub-repo flags can
be used to specify
+ the repository to push to. For GitHub repository --github-organisation
and --github-repo
+ flags can be used for the same purpose.
+
+ You can also add --production-image flag to switch to production image
(default is CI one)
+
+ Examples:
+
+ 'breeze push-image' or
+ 'breeze push-image --dockerhub-user user' to push to your private
registry or
+ 'breeze push-image --production-image' - to push production image or
+ 'breeze push-image --registry-cache' - to push to GitHub cache or
+ 'breeze push-image --registry-cache --github-organisation org' - for
other organisation
+
+ Flags:
+
+ -D, --dockerhub-user
+ DockerHub user used to pull, push and build images. Default: apache.
+
+ -H, --dockerhub-repo
+ DockerHub repository used to pull, push, build images. Default:
airflow.
+
+ -c, --registry-cache
+ If registry cache is enabled, pulls and pushes are done from the
registry cache in github.
+ You need to be logged in to the registry in order to be able to
pull/push from it and you
+ need to be committer to push to airflow registry.
+
+ -G, --github-organisation
+ GitHub organisation used to pull, push images when cache is used.
Default: apache.
+
+ -g, --github-repo
+ GitHub repository used to pull, push images when cache is used.
Default: airflow.
+
+ -v, --verbose
+ Show verbose information about executed commands (enabled by default
for running test).
+ Note that you can further increase verbosity and see all the
commands executed by breeze
+ by running 'export VERBOSE_COMMANDS="true"' before running breeze.
+
+
+
####################################################################################################
+
+
Detailed usage for command: initialize-local-virtualenv
breeze [FLAGS] initialize-local-virtualenv -- <EXTRA_ARGS>
@@ -1436,12 +1492,7 @@ This is the current syntax for `./breeze <./breeze>`_:
builds in the Docker cache are used instead.
****************************************************************************************************
- Flags for pushing Docker images (both CI and production)
-
- -u, --push-images
- After building - uploads the images to DockerHub
- It is useful in case you use your own DockerHub user to store images
and you want
- to build them locally. Note that you need to use 'docker login'
before you upload images.
+ Flags for pulling/pushing Docker images (both CI and production)
-D, --dockerhub-user
DockerHub user used to pull, push and build images. Default: apache.
@@ -1449,6 +1500,17 @@ This is the current syntax for `./breeze <./breeze>`_:
-H, --dockerhub-repo
DockerHub repository used to pull, push, build images. Default:
airflow.
+ -c, --registry-cache
+ If registry cache is enabled, pulls and pushes are done from the
registry cache in github.
+ You need to be logged in to the registry in order to be able to
pull/push from it and you
+ need to be committer to push to airflow registry.
+
+ -G, --github-organisation
+ GitHub organisation used to pull, push images when cache is used.
Default: apache.
+
+ -g, --github-repo
+ GitHub repository used to pull, push images when cache is used.
Default: airflow.
+
****************************************************************************************************
Flags for generation of the backport packages
diff --git a/breeze b/breeze
index 512eecd..bd5991f 100755
--- a/breeze
+++ b/breeze
@@ -791,13 +791,24 @@ function parse_arguments() {
echo
export FORWARD_CREDENTIALS="true"
shift 1 ;;
- -u|--push-images)
+ -c|--registry-cache)
echo
- echo "Pushing images to DockerHub"
+ echo "Use cache for the container registry"
echo
- export PUSH_IMAGES="true"
- export FORCE_BUILD_IMAGES="true"
+ export ENABLE_REGISTRY_CACHE="true"
shift ;;
+ -G|--github-organisation)
+ echo
+ echo "GitHub organisation"
+ echo
+ export GITHUB_ORGANISATION="${2}"
+ shift 2;;
+ -g|--github-repo)
+ echo
+ echo "GitHub repository"
+ echo
+ export GITHUB_REPOSITORY="${2}"
+ shift 2;;
-S|--version-suffix-for-pypi)
if [[ ${VERSION_SUFFIX_FOR_SVN} != "" ]]; then
echo
@@ -881,6 +892,10 @@ function parse_arguments() {
generate-requirements)
LAST_SUBCOMMAND="${1}"
COMMAND_TO_RUN="perform_generate_requirements"
+ # if you want to generate requirement - you want to build the image
too :)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ # and assume you want to build it no matter if it is needed
+ export FORCE_BUILD_IMAGES="true"
shift ;;
prepare-backport-packages)
LAST_SUBCOMMAND="${1}"
@@ -890,6 +905,11 @@ function parse_arguments() {
LAST_SUBCOMMAND="${1}"
COMMAND_TO_RUN="perform_prepare_backport_readme"
shift ;;
+ push-image)
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_push_image"
+ SKIP_CHECK_REMOTE_IMAGE="true"
+ shift ;;
initialize-local-virtualenv)
LAST_SUBCOMMAND="${1}"
echo "Initializing local virtualenv"
@@ -1077,12 +1097,13 @@ function prepare_usage() {
export USAGE_DOCKER_COMPOSE="Executes specified docker-compose command"
export USAGE_EXECUTE_COMMAND="Executes specified command in the container"
export USAGE_FLAGS="Shows all breeze's flags"
- export USAGE_PREPARE_BACKPORT_README="Prepares backport packages readme
files"
export USAGE_GENERATE_REQUIREMENTS="Generates pinned requirements for pip
dependencies"
export USAGE_INITIALIZE_LOCAL_VIRTUALENV="Initializes local virtualenv"
+ export USAGE_PREPARE_BACKPORT_README="Prepares backport packages readme
files"
+ export USAGE_PREPARE_BACKPORT_PACKAGES="Prepares backport packages"
+ export USAGE_PUSH_IMAGE="Pushes images to registry"
export USAGE_SETUP_AUTOCOMPLETE="Sets up autocomplete for breeze"
export USAGE_STOP="Stops the docker-compose environment"
- export USAGE_PREPARE_BACKPORT_PACKAGES="Prepares backport packages"
export USAGE_RESTART="Stops the docker-compose environment including DB
cleanup"
export USAGE_STATIC_CHECK="Performs selected static check for changed
files"
export USAGE_STATIC_CHECK_ALL_FILES="Performs selected static check for
all files"
@@ -1140,7 +1161,7 @@ $(flag_airflow_variants)
$(flag_choose_different_airflow_version)
$(flag_production_image)
$(flag_build_docker_images)
-$(flag_push_docker_images)
+$(flag_pull_push_docker_images)
$(flag_verbosity)
"
# shellcheck disable=SC2090
@@ -1278,6 +1299,31 @@ $(flag_verbosity)
"
# shellcheck disable=SC2090
export DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_PUSH_IMAGE="
+ Pushes images to docker registry. You can push the images to DockerHub
registry (default)
+ or to the GitHub cache registry (if --registry-cache flag is used).
+
+ For DockerHub pushes --dockerhub-user and --dockerhub-repo flags can be
used to specify
+ the repository to push to. For GitHub repository --github-organisation
and --github-repo
+ flags can be used for the same purpose.
+
+ You can also add --production-image flag to switch to production image
(default is CI one)
+
+ Examples:
+
+ '${CMDNAME} push-image' or
+ '${CMDNAME} push-image --dockerhub-user user' to push to your private
registry or
+ '${CMDNAME} push-image --production-image' - to push production image or
+ '${CMDNAME} push-image --registry-cache' - to push to GitHub cache or
+ '${CMDNAME} push-image --registry-cache --github-organisation org' - for
other organisation
+
+Flags:
+$(flag_pull_push_docker_images)
+$(flag_verbosity)
+"
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_PUSH_IMAGE
export DETAILED_USAGE_SETUP_AUTOCOMPLETE="
Sets up autocomplete for breeze commands. Once you do it you need to
re-enter the bash
shell and when typing breeze command <TAB> will provide autocomplete for
@@ -1639,19 +1685,26 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS}
"
}
-# Prints flags controlling docker push process
-function flag_push_docker_images() {
+# Prints flags controlling docker pull and push process
+function flag_pull_push_docker_images() {
echo "
--u, --push-images
- After building - uploads the images to DockerHub
- It is useful in case you use your own DockerHub user to store images
and you want
- to build them locally. Note that you need to use 'docker login' before
you upload images.
-
-D, --dockerhub-user
DockerHub user used to pull, push and build images. Default:
${_BREEZE_DEFAULT_DOCKERHUB_USER:=}.
-H, --dockerhub-repo
DockerHub repository used to pull, push, build images. Default:
${_BREEZE_DEFAULT_DOCKERHUB_REPO:=}.
+
+-c, --registry-cache
+ If registry cache is enabled, pulls and pushes are done from the
registry cache in github.
+ You need to be logged in to the registry in order to be able to
pull/push from it and you
+ need to be committer to push to airflow registry.
+
+-G, --github-organisation
+ GitHub organisation used to pull, push images when cache is used.
Default: ${_BREEZE_DEFAULT_GITHUB_ORGANISATION:=}.
+
+-g, --github-repo
+ GitHub repository used to pull, push images when cache is used.
Default: ${_BREEZE_DEFAULT_GITHUB_REPO:=}.
+
"
}
@@ -1718,8 +1771,8 @@ $(print_star_line)
$(flag_build_docker_images)
$(print_star_line)
- Flags for pushing Docker images (both CI and production)
-$(flag_push_docker_images)
+ Flags for pulling/pushing Docker images (both CI and production)
+$(flag_pull_push_docker_images)
$(print_star_line)
Flags for generation of the backport packages
@@ -1953,7 +2006,7 @@ function run_build_command {
rebuild_ci_image_if_needed
fi
;;
-
build_docs|perform_static_checks|perform_generate_requirements|perform_prepare_backport_readme|perform_prepare_backport_packages)
+
build_docs|perform_static_checks|perform_generate_requirements|perform_prepare_backport_readme|perform_prepare_backport_packages|perform_push_image)
prepare_ci_build
rebuild_ci_image_if_needed
;;
@@ -1981,6 +2034,7 @@ function run_build_command {
echo >&2
echo >&2 "ERROR: Unknown command to run ${COMMAND_TO_RUN}"
echo >&2
+ exit 1
;;
esac
}
@@ -2038,13 +2092,6 @@ function run_breeze_command {
run_static_checks "$@"
;;
build_image)
- if [[ ${PUSH_IMAGES} == "true" ]]; then
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- push_prod_images
- else
- push_ci_image
- fi
- fi
;;
cleanup_image)
remove_images
@@ -2058,6 +2105,13 @@ function run_breeze_command {
perform_prepare_backport_readme)
run_prepare_backport_readme "${@}"
;;
+ perform_push_image)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ push_prod_images
+ else
+ push_ci_image
+ fi
+ ;;
perform_initialize_local_virtualenv)
initialize_virtualenv
;;
diff --git a/breeze-complete b/breeze-complete
index 0928810..dc561db 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -89,6 +89,8 @@ EOF
_BREEZE_DEFAULT_DOCKERHUB_USER="apache"
_BREEZE_DEFAULT_DOCKERHUB_REPO="airflow"
+_BREEZE_DEFAULT_GITHUB_ORGANISATION="apache"
+_BREEZE_DEFAULT_GITHUB_REPO="airflow"
_BREEZE_SHORT_OPTIONS="
h p: b: i:
@@ -97,7 +99,7 @@ s x r
l a: t: d:
v y n q f
F P I E: C L
-D: R: u
+D: R: c g: G:
S: N:
"
@@ -108,7 +110,7 @@ kind-cluster-start kind-cluster-stop kind-cluster-recreate
skip-mounting-local-sources install-airflow-version:
install-airflow-reference: db-reset
verbose assume-yes assume-no assume-quit forward-credentials
force-build-images force-pull-images production-image extras:
force-clean-images use-local-cache
-dockerhub-user: dockerhub-repo: push-images
+dockerhub-user: dockerhub-repo: registry-cache github-organisation:
github-repo:
postgres-version: mysql-version:
version-suffix-for-pypi: version-suffix-for-svn:
additional-extras: additional-python-deps:
@@ -123,6 +125,7 @@ exec
generate-requirements
prepare-backport-readme
prepare-backport-packages
+push-image
initialize-local-virtualenv
setup-autocomplete
stop
@@ -193,6 +196,12 @@ function _get_known_values_breeze() {
-R | --dockerhub-repo)
_BREEZE_KNOWN_VALUES="${_BREEZE_DEFAULT_DOCKERHUB_REPO}"
;;
+ -G | --github-organisation)
+ _BREEZE_KNOWN_VALUES="${_BREEZE_DEFAULT_GITHUB_ORGANISATION}"
+ ;;
+ -g | --github-repo)
+ _BREEZE_KNOWN_VALUES="${_BREEZE_DEFAULT_GITHUB_REPO}"
+ ;;
*)
_BREEZE_KNOWN_VALUES=""
;;
diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh
index 3df4b0c..899ea50 100644
--- a/scripts/ci/_utils.sh
+++ b/scripts/ci/_utils.sh
@@ -83,6 +83,11 @@ function initialize_common_environment {
# Default branch name for triggered builds is the one configured in
default branch
export BRANCH_NAME=${BRANCH_NAME:=${DEFAULT_BRANCH}}
+ export GITHUB_ORGANISATION=${GITHUB_ORGANISATION:="apache"}
+ export GITHUB_REPO=${GITHUB_REPO:="airflow"}
+ export CACHE_REGISTRY=${CACHE_REGISTRY:="docker.pkg.github.com"}
+ export ENABLE_REGISTRY_CACHE=${ENABLE_REGISTRY_CACHE:="false"}
+
# Default port numbers for forwarded ports
export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="28080"}
export POSTGRES_HOST_PORT=${POSTGRES_HOST_PORT:="25433"}
@@ -92,10 +97,6 @@ function initialize_common_environment {
export POSTGRES_VERSION=${POSTGRES_VERSION:="9.6"}
export MYSQL_VERSION=${MYSQL_VERSION:="5.7"}
- # Do not push images by default (push them directly from the build script
on Dockerhub or when
- # --push-images flag is specified
- export PUSH_IMAGES=${PUSH_IMAGES:="false"}
-
# Whether base python images should be pulled from cache
export
PULL_PYTHON_BASE_IMAGES_FROM_CACHE=${PULL_PYTHON_BASE_IMAGES_FROM_CACHE:="true"}
@@ -1480,11 +1481,14 @@ function prepare_ci_build() {
export
AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
export
AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
export
AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
- if [[ ${CACHE_REGISTRY=""} != "" ]]; then
- echo "${CACHE_REGISTRY_PASSWORD}" | docker login \
- --username "${CACHE_REGISTRY_USERNAME}" \
- --password-stdin \
- "${CACHE_REGISTRY}"
+ if [[ ${ENABLE_REGISTRY_CACHE="false"} == "true" ]]; then
+ if [[ ${CACHE_REGISTRY_PASSWORD:=} != "" ]]; then
+ echo "${CACHE_REGISTRY_PASSWORD}" | docker login \
+ --username "${CACHE_REGISTRY_USERNAME}" \
+ --password-stdin \
+ "${CACHE_REGISTRY}"
+ fi
+ export
CACHE_IMAGE_PREFIX=${CACHE_IMAGE_PREFX:=${GITHUB_ORGANISATION}/${GITHUB_REPO}}
export
CACHED_AIRFLOW_CI_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/${AIRFLOW_CI_BASE_TAG}"
export
CACHED_PYTHON_BASE_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster"
else
@@ -1568,11 +1572,14 @@ function prepare_prod_build() {
export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}"
- if [[ ${CACHE_REGISTRY=""} != "" ]]; then
- echo "${CACHE_REGISTRY_PASSWORD}" | docker login \
- --username "${CACHE_REGISTRY_USERNAME}" \
- --password-stdin \
- "${CACHE_REGISTRY}"
+ if [[ ${ENABLE_REGISTRY_CACHE="false"} == "true" ]]; then
+ if [[ ${CACHE_REGISTRY_PASSWORD:=} != "" ]]; then
+ echo "${CACHE_REGISTRY_PASSWORD}" | docker login \
+ --username "${CACHE_REGISTRY_USERNAME}" \
+ --password-stdin \
+ "${CACHE_REGISTRY}"
+ fi
+ export
CACHE_IMAGE_PREFIX=${CACHE_IMAGE_PREFX:=${GITHUB_ORGANISATION}/${GITHUB_REPO}}
export
CACHED_AIRFLOW_PROD_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/${AIRFLOW_PROD_BASE_TAG}"
export
CACHED_AIRFLOW_PROD_BUILD_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/${AIRFLOW_PROD_BASE_TAG}-build"
export
CACHED_PYTHON_BASE_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster"
diff --git a/scripts/ci/ci_fix_ownership.sh b/scripts/ci/ci_fix_ownership.sh
index ef1437a..f8284db 100755
--- a/scripts/ci/ci_fix_ownership.sh
+++ b/scripts/ci/ci_fix_ownership.sh
@@ -19,6 +19,7 @@
#
# Fixes ownership for files created inside container (files owned by root will
be owned by host user)
#
+set -x
export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
@@ -28,6 +29,14 @@ export
PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
export AIRFLOW_CI_IMAGE=\
${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci
+export AIRFLOW_IMAGE=${AIRFLOW_CI_IMAGE}
+export WEBSERVER_HOST_PORT=28080
+HOST_USER_ID="$(id -ur)"
+HOST_GROUP_ID="$(id -gr)"
+
+export HOST_USER_ID
+export HOST_GROUP_ID
+
docker-compose \
-f "${MY_DIR}/docker-compose/base.yml" \
-f "${MY_DIR}/docker-compose/local.yml" \
diff --git a/scripts/ci/in_container/_in_container_utils.sh
b/scripts/ci/in_container/_in_container_utils.sh
index 12bea89..7a9a523 100644
--- a/scripts/ci/in_container/_in_container_utils.sh
+++ b/scripts/ci/in_container/_in_container_utils.sh
@@ -93,7 +93,7 @@ function in_container_cleanup_pycache() {
#
function in_container_fix_ownership() {
set +o pipefail
- sudo find "${AIRFLOW_SOURCES}" -user root -print0 \
+ sudo find "${AIRFLOW_SOURCES}" -print0 -user root \
| sudo xargs --null chown -v "${HOST_USER_ID}.${HOST_GROUP_ID}"
--no-dereference >/dev/null 2>&1
set -o pipefail
}
