This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch extract-basic-tests-to-separate-workflow in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ceab36d28e450d4a963bb678b444be6b88148594 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Sat Mar 16 11:01:28 2024 +0100 Extract basic tests to separate workflow Similarly as with other extractions - this one will extract all the tests that need just basic "build-info" dependencies to a separate workflow. This will allow to keep the ci.yml better organised - eventually consisting of just calling other workflows. It also assigns all the basic checks to be run on the public ASF runners. There is no point in using our self-hosted runners to run those tests, they are usually pretty fast even on a small public runner (and they are not on a critical path for elapsed time of the workflow run - no other tests depend on them). --- .github/workflows/basic-tests.yml | 329 ++++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 290 ++------------------------------- 2 files changed, 343 insertions(+), 276 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml new file mode 100644 index 0000000000..03d282bb01 --- /dev/null +++ b/.github/workflows/basic-tests.yml @@ -0,0 +1,329 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Basic tests +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + runs-www-tests: + description: "Whether to run WWW tests (true/false)" + required: true + type: string + needs-api-codegen: + description: "Whether to run API codegen (true/false)" + required: true + type: string + basic-checks-only: + description: "Whether to run only basic checks (true/false)" + required: true + type: string + skip-pre-commits: + description: "Whether to skip pre-commits (true/false)" + required: true + type: string + default-python-version: + description: Which version of python should be used get CI image + required: true + type: string + breeze-python-version: + description: > + Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds) + required: true + type: string + canary-run: + description: "Whether to run canary tests (true/false)" + required: true + type: string + latest-versions-only: + description: "Whether to run only latest version checks (true/false)" + required: true + type: string +jobs: + run-breeze-tests: + timeout-minutes: 10 + name: Breeze unit tests + runs-on: ["ubuntu-22.04"] + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - uses: actions/checkout@v4 + with: + # Need to fetch all history for selective checks tests + fetch-depth: 0 + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - uses: actions/setup-python@v5 + with: + python-version: "${{ inputs.default-python-version }}" + cache: 'pip' + cache-dependency-path: ./dev/breeze/setup* + - run: pip install --editable ./dev/breeze/ + - run: python -m pytest -n auto --color=yes + working-directory: ./dev/breeze/ + + tests-www: + timeout-minutes: 10 + name: React WWW tests + runs-on: ["ubuntu-22.04"] + if: inputs.run-www-tests == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - name: "Setup node" + uses: actions/setup-node@v4 + with: + node-version: 21 + - name: "Cache eslint" + uses: actions/cache@v4 + with: + path: 'airflow/www/node_modules' + key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }} + - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive + - run: yarn --cwd airflow/www/ run test + env: + FORCE_COLOR: 2 + + test-openapi-client: + timeout-minutes: 10 + name: "Test OpenAPI client" + runs-on: ["ubuntu-22.04"] + if: inputs.needs-api-codegen == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + fetch-depth: 2 + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + repository: "apache/airflow-client-python" + fetch-depth: 1 + persist-credentials: false + path: ./airflow-client-python + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + python-version: ${{ inputs.breeze-python-version }} + - name: "Generate client with breeze" + run: > + breeze release-management prepare-python-client --package-format both + --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python + - name: "Show diff" + run: git diff --color HEAD + working-directory: ./airflow-client-python + - name: Install hatch + run: | + python -m pip install --upgrade pipx + pipx ensurepath + pipx install hatch --force + - name: Run tests + run: hatch run run-coverage + env: + HATCH_ENV: "test" + working-directory: ./clients/python + - name: "Install Airflow in editable mode with fab for webserver tests" + run: pip install -e ".[fab]" + - name: "Install Python client" + run: pip install ./dist/apache_airflow_client-*.whl + - name: "Initialize Airflow DB and start webserver" + run: | + airflow db init + # Let scheduler runs a few loops and get all DAG files from example DAGs serialized to DB + airflow scheduler --num-runs 100 + airflow users create --username admin --password admin --firstname Admin --lastname Admin \ + --role Admin --email ad...@example.org + killall python || true # just in case there is a webserver running in the background + nohup airflow webserver --port 8080 & + echo "Started webserver" + env: + AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.session,airflow.api.auth.backend.basic_auth + AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True" + AIRFLOW__CORE__LOAD_EXAMPLES: "True" + AIRFLOW_HOME: "${{ github.workspace }}/airflow_home" + - name: "Waiting for the webserver to be available" + run: | + timeout 30 bash -c 'until nc -z $0 $1; do echo "sleeping"; sleep 1; done' localhost 8080 + sleep 5 + - name: "Run test python client" + run: python ./clients/python/test_python_client.py + env: + FORCE_COLOR: "standard" + - name: "Stop running webserver" + run: killall python || true # just in case there is a webserver running in the background + if: always() + - name: "Upload python client packages" + uses: actions/upload-artifact@v4 + with: + name: python-client-packages + path: ./dist/apache_airflow_client-* + retention-days: 7 + if-no-files-found: error + - name: "Upload logs from failed tests" + uses: actions/upload-artifact@v4 + if: failure() + with: + name: python-client-failed-logs + path: "${{ github.workspace }}/airflow_home/logs" + retention-days: 7 + + # Those checks are run if no image needs to be built for checks. This is for simple changes that + # Do not touch any of the python code or any of the important files that might require building + # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host + static-checks-basic-checks-only: + timeout-minutes: 30 + name: "Static checks: basic checks only" + runs-on: ["ubuntu-22.04"] + if: info.basic-checks-only == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - name: "Setup python" + uses: actions/setup-python@v5 + with: + python-version: "${{ inputs.default-python-version }}" + cache: 'pip' + cache-dependency-path: ./dev/breeze/setup* + - name: "Install Breeze" + uses: ./.github/actions/breeze + id: breeze + - name: Cache pre-commit envs + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + # yamllint disable-line rule:line-length + key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: "\ + pre-commit-${{steps.breeze.outputs.host-python-version}}-\ + ${{ hashFiles('.pre-commit-config.yaml') }}\n + pre-commit-${{steps.breeze.outputs.host-python-version}}-" + - name: Fetch incoming commit ${{ github.sha }} with its parent + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false + - name: "Static checks: basic checks only" + run: > + breeze static-checks --show-diff-on-failure --color always --initialize-environment + --skip-image-upgrade-check --commit-ref "${{ github.sha }}" + env: + VERBOSE: "false" + SKIP_BREEZE_PRE_COMMITS: "true" + SKIP: ${{ inputs.skip-pre-commits }} + COLUMNS: "250" + + test-git-clone-on-windows: + timeout-minutes: 5 + name: "Test git clone on Windows" + runs-on: ["windows-latest"] + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + fetch-depth: 2 + persist-credentials: false + + upgrade-check: + timeout-minutes: 45 + name: "Upgrade checks" + runs-on: ["ubuntu-22.04"] + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + # Install python from scratch. No cache used. We always want to have fresh version of everything + - uses: actions/setup-python@v5 + with: + python-version: "${{ inputs.default-python-version }}" + - name: "Install latest pre-commit" + run: pip install pre-commit + - name: "Autoupdate all pre-commits" + run: pre-commit autoupdate + - name: "Run automated upgrade for black" + run: > + pre-commit run + --all-files --show-diff-on-failure --color always --verbose + --hook-stage manual + update-black-version + if: always() + - name: "Run automated upgrade for build dependencies" + run: > + pre-commit run + --all-files --show-diff-on-failure --color always --verbose + --hook-stage manual + update-build-dependencies + if: always() + - name: "Run automated upgrade for chart dependencies" + run: > + pre-commit run + --all-files --show-diff-on-failure --color always --verbose + --hook-stage manual + update-chart-dependencies + if: always() + # For UV we are not failing the upgrade installers check if it is updated because + # it is upgraded very frequently, so we want to manually upgrade it rather than + # get notified about it - until it stabilizes in 1.* version + - name: "Run automated upgrade for uv (open to see if new version is updated)" + run: > + pre-commit run + --all-files --show-diff-on-failure --color always --verbose + --hook-stage manual update-installers || true + if: always() + env: + UPGRADE_UV: "true" + UPGRADE_PIP: "false" + - name: "Run automated upgrade for pip" + run: > + pre-commit run + --all-files --show-diff-on-failure --color always --verbose + --hook-stage manual update-installers + if: always() + env: + UPGRADE_UV: "false" + UPGRADE_PIP: "true" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 620b0bfac4..9ab238b2c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -236,6 +236,20 @@ jobs: - name: "Check that image builds quickly" run: breeze shell --max-time 120 + basic-tests: + name: "Basic tests" + needs: [build-info] + uses: ./.github/workflows/basic-tests.yml + with: + run-www-tests: ${{needs.build-info.outputs.run-www-tests}} + needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} + default-python-version: ${{needs.build-info.outputs.default-python-version}} + breeze-python-version: ${{needs.build-info.outputs.breeze-python-version}} + basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} + skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} + canary-run: ${{needs.build-info.outputs.canary-run}} + latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} + build-ci-images: name: > ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} @@ -262,154 +276,6 @@ jobs: constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.cache-directive }} - run-breeze-tests: - timeout-minutes: 10 - name: Breeze unit tests - runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} - needs: [build-info] - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@v4 - with: - # Need to fetch all history for selective checks tests - fetch-depth: 0 - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - uses: actions/setup-python@v5 - with: - python-version: "${{needs.build-info.outputs.default-python-version}}" - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: pip install --editable ./dev/breeze/ - - run: python -m pytest -n auto --color=yes - working-directory: ./dev/breeze/ - - tests-www: - timeout-minutes: 10 - name: React WWW tests - runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} - needs: [build-info] - if: needs.build-info.outputs.run-www-tests == 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Setup node" - uses: actions/setup-node@v4 - with: - node-version: 21 - - name: "Cache eslint" - uses: actions/cache@v4 - with: - path: 'airflow/www/node_modules' - key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }} - - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive - - run: yarn --cwd airflow/www/ run test - env: - FORCE_COLOR: 2 - - - test-openapi-client: - timeout-minutes: 10 - name: "Test OpenAPI client" - runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} - needs: [build-info] - if: needs.build-info.outputs.needs-api-codegen == 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - fetch-depth: 2 - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - repository: "apache/airflow-client-python" - fetch-depth: 1 - persist-credentials: false - path: ./airflow-client-python - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: "Generate client with breeze" - run: > - breeze release-management prepare-python-client --package-format both - --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python - - name: "Show diff" - run: git diff --color HEAD - working-directory: ./airflow-client-python - - name: Install hatch - run: | - python -m pip install --upgrade pipx - pipx ensurepath - pipx install hatch --force - - name: Run tests - run: hatch run run-coverage - env: - HATCH_ENV: "test" - working-directory: ./clients/python - - name: "Install Airflow in editable mode with fab for webserver tests" - run: pip install -e ".[fab]" - - name: "Install Python client" - run: pip install ./dist/apache_airflow_client-*.whl - - name: "Initialize Airflow DB and start webserver" - run: | - airflow db init - # Let scheduler runs a few loops and get all DAG files from example DAGs serialized to DB - airflow scheduler --num-runs 100 - airflow users create --username admin --password admin --firstname Admin --lastname Admin \ - --role Admin --email ad...@example.org - killall python || true # just in case there is a webserver running in the background - nohup airflow webserver --port 8080 & - echo "Started webserver" - env: - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.session,airflow.api.auth.backend.basic_auth - AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True" - AIRFLOW__CORE__LOAD_EXAMPLES: "True" - - name: "Waiting for the webserver to be available" - run: | - timeout 30 bash -c 'until nc -z $0 $1; do echo "sleeping"; sleep 1; done' localhost 8080 - sleep 5 - - name: "Run test python client" - run: python ./clients/python/test_python_client.py - env: - FORCE_COLOR: "standard" - - name: "Stop running webserver" - run: killall python || true # just in case there is a webserver running in the background - if: always() - - name: "Upload python client packages" - uses: actions/upload-artifact@v4 - with: - name: python-client-packages - path: ./dist/apache_airflow_client-* - retention-days: 7 - if-no-files-found: error - - test-git-clone-on-windows: - timeout-minutes: 5 - name: "Test git clone on Windows" - runs-on: "windows-latest" - needs: [build-info] - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - fetch-depth: 2 - persist-credentials: false - wait-for-ci-images: timeout-minutes: 120 name: "Wait for CI images" @@ -570,78 +436,6 @@ jobs: DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} RUFF_FORMAT: "github" - upgrade-check: - timeout-minutes: 45 - name: "Upgrade checks" - runs-on: ["ubuntu-22.04"] - needs: [build-info] - env: - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" - if: > - needs.build-info.outputs.canary-run == 'true' && - needs.build-info.outputs.latest-versions-only != 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - # Install python from scratch. No cache used. We always want to have fresh version of everything - - uses: actions/setup-python@v5 - with: - python-version: "${{needs.build-info.outputs.default-python-version}}" - - name: "Install latest pre-commit" - run: pip install pre-commit - - name: "Autoupdate all pre-commits" - run: pre-commit autoupdate - - name: "Run automated upgrade for black" - run: > - pre-commit run - --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual - update-black-version - if: always() - - name: "Run automated upgrade for build dependencies" - run: > - pre-commit run - --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual - update-build-dependencies - if: always() - - name: "Run automated upgrade for chart dependencies" - run: > - pre-commit run - --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual - update-chart-dependencies - if: always() - # For UV we are not failing the upgrade installers check if it is updated because - # it is upgraded very frequently, so we want to manually upgrade it rather than - # get notified about it - until it stabilizes in 1.* version - - name: "Run automated upgrade for uv (open to see if new version is updated)" - run: > - pre-commit run - --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual update-installers || true - if: always() - env: - UPGRADE_UV: "true" - UPGRADE_PIP: "false" - - name: "Run automated upgrade for pip" - run: > - pre-commit run - --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual update-installers - if: always() - env: - UPGRADE_UV: "false" - UPGRADE_PIP: "true" - mypy: name: "MyPy tests" uses: ./.github/workflows/mypy.yml @@ -659,62 +453,6 @@ jobs: needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} if: needs.build-info.outputs.latest-versions-only != 'true' - # Those checks are run if no image needs to be built for checks. This is for simple changes that - # Do not touch any of the python code or any of the important files that might require building - # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host - static-checks-basic-checks-only: - timeout-minutes: 30 - name: "Static checks: basic checks only" - runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} - needs: [build-info] - env: - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - if: needs.build-info.outputs.basic-checks-only == 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: "${{needs.build-info.outputs.default-python-version}}" - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - name: "Install Breeze" - uses: ./.github/actions/breeze - id: breeze - - name: Cache pre-commit envs - uses: actions/cache@v4 - with: - path: ~/.cache/pre-commit - # yamllint disable-line rule:line-length - key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: "\ - pre-commit-${{steps.breeze.outputs.host-python-version}}-\ - ${{ hashFiles('.pre-commit-config.yaml') }}\n - pre-commit-${{steps.breeze.outputs.host-python-version}}-" - - name: Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@v4 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: "Static checks: basic checks only" - run: > - breeze static-checks --show-diff-on-failure --color always --initialize-environment - --skip-image-upgrade-check --commit-ref "${{ github.sha }}" - env: - VERBOSE: "false" - SKIP_BREEZE_PRE_COMMITS: "true" - SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} - COLUMNS: "250" - build-docs: timeout-minutes: 60 name: "Build docs"