This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c92954418a Breeze is installed via pipx in CI (#23023)
c92954418a is described below
commit c92954418a21dcafcb0b87864ffcb77a67a707bb
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Apr 15 12:19:37 2022 +0200
Breeze is installed via pipx in CI (#23023)
The new Breeze needs to be installed via pipx in CI - except when
it is unit-tested. Also it needs to be added to PATH.
This change adds a very small script that is used in all places
where Breeze needs to be installed.
---
.github/workflows/build-images.yml | 4 +--
.github/workflows/ci.yml | 55 +++++++++++++-------------------
docker_tests/requirements.txt | 3 ++
scripts/ci/images/ci_run_docker_tests.py | 13 ++++++--
scripts/ci/install_breeze.sh | 24 ++++++++++++++
5 files changed, 63 insertions(+), 36 deletions(-)
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index 3ab937bdc3..79fc2ad154 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -221,7 +221,7 @@ jobs:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Build/Push CI image ${{ matrix.python-version }}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -312,7 +312,7 @@ jobs:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image for PROD build: ${{ matrix.python-version }}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cd51fbcd84..129b1c7193 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -305,7 +305,7 @@ jobs:
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Retrieve DEFAULTS from the _initialization.sh"
# We cannot "source" the script here because that would be a security
problem (we cannot run
@@ -384,7 +384,7 @@ jobs:
cut -d "=" -f 3 | sed s'/["}]//g')
echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
run: airflow-freespace
@@ -504,20 +504,11 @@ jobs:
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
- cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
- - name: "Free space"
- run: airflow-freespace
- - name: "Cache virtualenv environment"
- uses: actions/cache@v2
- with:
- path: '.build/.docker_venv'
- key: ${{ runner.os }}-docker-venv-${{
hashFiles('scripts/ci/images/ci_run_docker_tests.py') }}
+ cache-dependency-path: ./dev/requirements.txt
- name: "Test examples of PROD image building"
- working-directory: docs/docker-stack/docker-examples
run: >
- python -m pytest
../../../docker_tests/test_examples_of_prod_image_building.py
- -n auto --color=yes
+ python -m pip install -r ./docker_tests/requirements.txt &&
+ python -m pytest
docker_tests/test_examples_of_prod_image_building.py -n auto --color=yes
test-docker-compose-quick-start:
timeout-minutes: 60
@@ -537,7 +528,7 @@ jobs:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Cache virtualenv environment"
@@ -568,7 +559,7 @@ jobs:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Cache virtualenv environment"
@@ -613,7 +604,7 @@ jobs:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -676,7 +667,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: >
Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v2
@@ -726,7 +717,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -785,7 +776,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -841,7 +832,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -895,7 +886,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -960,7 +951,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -1028,7 +1019,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -1095,7 +1086,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -1158,7 +1149,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -1231,7 +1222,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
if: github.ref == 'refs/heads/v1-10-test'
run: |
echo "ISSUE_ID=10128" >> $GITHUB_ENV
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -1325,7 +1316,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Cache virtualenv environment"
@@ -1386,7 +1377,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Get all PROD images: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -1447,7 +1438,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Get all PROD images: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -1519,7 +1510,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: >
@@ -1598,7 +1589,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: python -m pip install --editable ./dev/breeze/
+ - run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: airflow-freespace
- name: "Start ARM instance"
diff --git a/docker_tests/requirements.txt b/docker_tests/requirements.txt
new file mode 100644
index 0000000000..1f79a84821
--- /dev/null
+++ b/docker_tests/requirements.txt
@@ -0,0 +1,3 @@
+pytest
+pytest-xdist
+requests
diff --git a/scripts/ci/images/ci_run_docker_tests.py
b/scripts/ci/images/ci_run_docker_tests.py
index 6fce6e679c..b99a17bb40 100755
--- a/scripts/ci/images/ci_run_docker_tests.py
+++ b/scripts/ci/images/ci_run_docker_tests.py
@@ -23,7 +23,7 @@ import sys
from pathlib import Path
from typing import List
-AIRFLOW_SOURCE = Path(__file__).resolve().parent.parent.parent
+AIRFLOW_SOURCE = Path(__file__).resolve().parents[3]
BUILD_CACHE_DIR = AIRFLOW_SOURCE / ".build"
CBLUE = '\033[94m'
@@ -62,7 +62,16 @@ def create_virtualenv():
run_verbose([sys.executable, "-m", "venv", str(virtualenv_path)])
python_bin = virtualenv_path / "bin" / "python"
- run_verbose([str(python_bin), "-m", "pip", "install", "pytest",
"pytest-xdist", "requests"])
+ run_verbose(
+ [
+ str(python_bin),
+ "-m",
+ "pip",
+ "install",
+ "-r",
+ str(AIRFLOW_SOURCE / "docker_tests" / "requirements.txt"),
+ ]
+ )
return python_bin
diff --git a/scripts/ci/install_breeze.sh b/scripts/ci/install_breeze.sh
new file mode 100755
index 0000000000..4055d2272b
--- /dev/null
+++ b/scripts/ci/install_breeze.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -euxo pipefail
+
+cd "$( dirname "${BASH_SOURCE[0]}" )/../../"
+
+python -m pip install pipx
+python -m pipx install --editable ./dev/breeze/
+echo '/home/runner/.local/bin' >> "${GITHUB_PATH}"
