This is an automated email from the ASF dual-hosted git repository. kaxilnaik pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f7ab31f12217c926fa56e915b864d763371d30a8 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Dec 14 15:10:06 2020 +0100 Allows to install Airflow in Breeze from PIP with configurable extras (#13055) The extras configured by --extras Breeze switch are now passed to pip install command in case airfow is installed via --install-airflow-version or --install-airflow-reference switch. (cherry picked from commit 26c685425996a2e8127d349a7b98ec92c74ee3f2) --- .github/workflows/ci.yml | 2 +- breeze | 10 ++--- .../ci/{libraries => docker-compose}/_docker.env | 52 ++++++++++++++++------ scripts/ci/docker-compose/base.yml | 43 ++---------------- scripts/ci/docker-compose/local-prod.yml | 9 +--- scripts/ci/libraries/_initialization.sh | 5 +-- scripts/ci/static_checks/check_license.sh | 2 +- scripts/ci/tools/ci_clear_tmp.sh | 2 +- scripts/ci/tools/ci_fix_ownership.sh | 2 +- scripts/in_container/_in_container_utils.sh | 7 --- scripts/in_container/entrypoint_ci.sh | 12 ++--- 11 files changed, 58 insertions(+), 88 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb06e0d..33fb5d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,6 +207,7 @@ jobs: runs-on: ubuntu-20.04 needs: [build-info, ci-images] env: + SKIP: "pylint" MOUNT_LOCAL_SOURCES: "true" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} if: needs.build-info.outputs.basic-checks-only == 'false' @@ -673,7 +674,6 @@ jobs: BACKEND: postgres RUN_TESTS: "true" RUNTIME: "kubernetes" - ENABLE_KIND_CLUSTER: "true" PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" KUBERNETES_MODE: "${{ matrix.kubernetes-mode }}" KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}" diff --git a/breeze b/breeze index 2565ad5..f6a45a5 100755 --- a/breeze +++ b/breeze @@ -58,7 +58,6 @@ export EXTRA_STATIC_CHECK_OPTIONS # SCREEN_WIDTH # MOUNT_LOCAL_SOURCES # FORCE_PULL_IMAGES -# ENABLE_KIND_CLUSTER # FORWARD_CREDENTIALS # DB_RESET # START_AIRFLOW @@ -101,9 +100,6 @@ function breeze::setup_default_breeze_constants() { # This can be overridden by '--force-pull-images' flag export FORCE_PULL_IMAGES="false" - # Do not enable Kind Kubernetes cluster by default - export ENABLE_KIND_CLUSTER="false" - # Forward common host credentials to docker (gcloud, aws etc.). export FORWARD_CREDENTIALS="false" @@ -824,7 +820,9 @@ function breeze::parse_arguments() { INSTALL_AIRFLOW_VERSION="${2}" # Reference is mutually exclusive with version INSTALL_AIRFLOW_REFERENCE="" - echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION}" + # Skip mounting local sources when airflow is installed from remote + INSTALL_PROVIDERS_FROM_SOURCES="false" + echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION} and skip mounting sources" echo shift 2 ;; @@ -832,6 +830,8 @@ function breeze::parse_arguments() { INSTALL_AIRFLOW_REFERENCE="${2}" # Reference is mutually exclusive with version INSTALL_AIRFLOW_VERSION="" + # Skip mounting local sources when airflow is installed from remote + INSTALL_PROVIDERS_FROM_SOURCES="false" echo "Installs Airflow from reference: ${INSTALL_AIRFLOW_REFERENCE}" echo shift 2 diff --git a/scripts/ci/libraries/_docker.env b/scripts/ci/docker-compose/_docker.env similarity index 76% rename from scripts/ci/libraries/_docker.env rename to scripts/ci/docker-compose/_docker.env index 6bc53dc..1ab933c 100644 --- a/scripts/ci/libraries/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -14,25 +14,49 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -INSTALL_AIRFLOW_VERSION -PYTHONDONTWRITEBYTECODE -VERBOSE -VERBOSE_COMMANDS +AIRFLOW_CI_IMAGE +AIRFLOW_EXTRAS +BACKEND +BACKPORT_PACKAGES +BREEZE +CI +CI_BUILD_ID +CI_JOB_ID +CI_EVENT_TYPE +CI_TARGET_REPO +CI_TARGET_BRANCH +COMMIT_SHA +DB_RESET +DEFAULT_CONSTRAINTS_BRANCH +DISABLE_RBAC +ENABLED_INTEGRATIONS +ENABLED_SYSTEMS +GITHUB_REGISTRY_PULL_IMAGE_TAG HOST_USER_ID HOST_GROUP_ID HOST_OS HOST_HOME HOST_AIRFLOW_SOURCES -PYTHON_MAJOR_MINOR_VERSION -BACKEND -VERSION_SUFFIX_FOR_PYPI -VERSION_SUFFIX_FOR_SVN -PRINT_INFO_FROM_SCRIPTS -CI +INIT_SCRIPT_FILE +INSTALL_AIRFLOW_VERSION +INSTALL_PROVIDERS_FROM_SOURCES +INSTALL_PACKAGES_FROM_DIST +ISSUE_ID LOAD_DEFAULT_CONNECTIONS LOAD_EXAMPLES -DEFAULT_CONSTRAINTS_BRANCH -GITHUB_REGISTRY_PULL_IMAGE_TAG -POSTGRES_VERSION MYSQL_VERSION -DISABLE_RBAC +NUM_RUNS +PACKAGE_FORMAT +POSTGRES_VERSION +PRINT_INFO_FROM_SCRIPTS +PYTHONDONTWRITEBYTECODE +PYTHON_MAJOR_MINOR_VERSION +RUN_TESTS +RUN_INTEGRATION_TESTS +RUN_SYSTEM_TESTS +START_AIRFLOW +TEST_TYPE +VERBOSE +VERBOSE_COMMANDS +VERSION_SUFFIX_FOR_PYPI +VERSION_SUFFIX_FOR_SVN diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 96bf0f6..f4f9851 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -24,47 +24,10 @@ services: - ADDITIONAL_PATH=~/.local/bin - CELERY_BROKER_URLS=amqp://guest:guest@rabbitmq:5672,redis://redis:6379/0 - KUBECONFIG=/files/.kube/config - - BACKEND - - CI - - CI_BUILD_ID - - CI_JOB_ID - - CI_EVENT_TYPE - - CI_TARGET_REPO - - CI_TARGET_BRANCH - - COMMIT_SHA - - RUN_TESTS - - VERBOSE - - VEROSE_COMMANDS - - AIRFLOW_CI_IMAGE - - ENABLE_KIND_CLUSTER - - ENABLED_INTEGRATIONS - - RUN_INTEGRATION_TESTS - - TEST_TYPE - - GITHUB_TOKEN - - GITHUB_REPOSITORY - - ISSUE_ID - - NUM_RUNS - - BREEZE - - INSTALL_AIRFLOW_VERSION - - DB_RESET - - START_AIRFLOW - - LOAD_EXAMPLES - - LOAD_DEFAULT_CONNECTIONS - - INSTALL_WHEELS - - DISABLE_RBAC - - ENABLED_SYSTEMS - - RUN_SYSTEM_TESTS - - PYTHON_MAJOR_MINOR_VERSION - - HOST_USER_ID - - HOST_GROUP_ID - - HOST_HOME=${HOME} - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES} - - HOST_OS - - PYTHONDONTWRITEBYTECODE - - INIT_SCRIPT_FILE - - GITHUB_REGISTRY_PULL_IMAGE_TAG - - POSTGRES_VERSION - - MYSQL_VERSION + - HOST_HOME=${HOME} + env_file: + - _docker.env volumes: # Pass docker to inside of the container so that Kind and Moto tests can use it. - /var/run/docker.sock:/var/run/docker.sock diff --git a/scripts/ci/docker-compose/local-prod.yml b/scripts/ci/docker-compose/local-prod.yml index 5a4bc8a..79476d0 100644 --- a/scripts/ci/docker-compose/local-prod.yml +++ b/scripts/ci/docker-compose/local-prod.yml @@ -29,16 +29,9 @@ services: - ../../../.github:/opt/airflow/.github:cached - ../../../.inputrc:/root/.inputrc:cached - ../../../.kube:/root/.kube:cached - - ../../../dist:/dist:cached - ../../../scripts/in_container/prod/entrypoint_prod.sh:/entrypoint:cached - ../../../setup.cfg:/opt/airflow/setup.cfg:cached - ../../../setup.py:/opt/airflow/setup.py:cached - ../../../tests:/opt/airflow/tests:cached - ../../../tmp:/tmp:cached - environment: - - HOST_USER_ID - - HOST_GROUP_ID - - HOST_HOME=${HOME} - - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES} - - HOST_OS - - PYTHONDONTWRITEBYTECODE + - ../../../metastore_browser:/opt/airflow/metastore_browser:cached diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index ff7abe9..50cdbf0 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -258,7 +258,7 @@ function initialization::initialize_mount_variables() { EXTRA_DOCKER_FLAGS+=( "--rm" - "--env-file" "${AIRFLOW_SOURCES}/scripts/ci/libraries/_docker.env" + "--env-file" "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" ) export EXTRA_DOCKER_FLAGS } @@ -404,8 +404,6 @@ function initialization::initialize_version_suffixes_for_package_building() { # Determine versions of kubernetes cluster and tools used function initialization::initialize_kubernetes_variables() { - # By default we assume the kubernetes cluster is not being started - export ENABLE_KIND_CLUSTER=${ENABLE_KIND_CLUSTER:="false"} # Currently supported versions of Kubernetes CURRENT_KUBERNETES_VERSIONS+=("v1.18.6" "v1.17.5" "v1.16.9") export CURRENT_KUBERNETES_VERSIONS @@ -679,7 +677,6 @@ function initialization::make_constants_read_only() { readonly HOST_HOME readonly HOST_OS - readonly ENABLE_KIND_CLUSTER readonly KUBERNETES_MODE readonly KUBERNETES_VERSION readonly KIND_VERSION diff --git a/scripts/ci/static_checks/check_license.sh b/scripts/ci/static_checks/check_license.sh index a185a5a..03858e3 100755 --- a/scripts/ci/static_checks/check_license.sh +++ b/scripts/ci/static_checks/check_license.sh @@ -33,7 +33,7 @@ function run_check_license() { # We mount ALL airflow files for the licence check. We want to check them all! if ! docker run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \ --user "$(id -ur):$(id -gr)" \ - --rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/libraries/_docker.env" \ + --rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ apache/airflow:apache-rat-2020.07.10-0.13 \ --exclude-file /opt/airflow/.rat-excludes \ --d /opt/airflow | tee "${AIRFLOW_SOURCES}/logs/rat-results.txt" ; then diff --git a/scripts/ci/tools/ci_clear_tmp.sh b/scripts/ci/tools/ci_clear_tmp.sh index 1806c96..d367967 100755 --- a/scripts/ci/tools/ci_clear_tmp.sh +++ b/scripts/ci/tools/ci_clear_tmp.sh @@ -29,6 +29,6 @@ read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker docker run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ --rm \ - --env-file "${AIRFLOW_SOURCES}/scripts/ci/libraries/_docker.env" \ + --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ "${AIRFLOW_CI_IMAGE}" \ -c /opt/airflow/scripts/in_container/run_clear_tmp.sh diff --git a/scripts/ci/tools/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh index 2e3a909..2d57d65 100755 --- a/scripts/ci/tools/ci_fix_ownership.sh +++ b/scripts/ci/tools/ci_fix_ownership.sh @@ -35,6 +35,6 @@ read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker docker run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ --rm \ - --env-file "${AIRFLOW_SOURCES}/scripts/ci/libraries/_docker.env" \ + --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ "${AIRFLOW_CI_IMAGE}" \ -c /opt/airflow/scripts/in_container/run_fix_ownership.sh diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index e2410fd..1dbcdd4 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -276,13 +276,6 @@ function uninstall_airflow_and_providers() { uninstall_airflow } -function install_all_airflow_dependencies() { - echo - echo "Installing dependencies from 'all' extras" - echo - pip install ".[all]" >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1 -} - function install_released_airflow_version() { local version="${1}" local extras="${2}" diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index 55962aa..ba9c2ee 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -108,23 +108,23 @@ elif [[ ${INSTALL_AIRFLOW_VERSION} == "none" ]]; then uninstall_airflow_and_providers elif [[ ${INSTALL_AIRFLOW_VERSION} == "wheel" ]]; then echo - echo "Install airflow from wheel package with [all] extras but uninstalling providers." + echo "Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers." echo uninstall_airflow_and_providers - install_airflow_from_wheel "[all]" + install_airflow_from_wheel "[${AIRFLOW_EXTRAS}]" uninstall_providers elif [[ ${INSTALL_AIRFLOW_VERSION} == "sdist" ]]; then echo - echo "Install airflow from sdist package with [all] extras but uninstalling providers." + echo "Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers." echo uninstall_airflow_and_providers - install_airflow_from_sdist "[all]" + install_airflow_from_sdist "[${AIRFLOW_EXTRAS}]" uninstall_providers else echo - echo "Install airflow from PyPI including [all] extras" + echo "Install airflow from PyPI including [${AIRFLOW_EXTRAS}] extras" echo - install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[all]" + install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[${AIRFLOW_EXTRAS}]" fi if [[ ${INSTALL_PACKAGES_FROM_DIST=} == "true" ]]; then echo
