potiuk commented on a change in pull request #8393:
URL: https://github.com/apache/airflow/pull/8393#discussion_r410725421
##########
File path: .github/workflows/ci.yml
##########
@@ -17,272 +17,275 @@
#
---
-name: Airflow
+name: CI Build
on:
+ schedule:
+ - cron: '10 2 * * *'
push:
- branches: ['*']
+ branches: ['master', 'v1-10-test', 'v1-10-stable']
pull_request:
- branches: [master]
+ branches: ['master', 'v1-10-test', 'v1-10-stable']
env:
- BUILD_ID: ${{github.sha }}
MOUNT_LOCAL_SOURCES: "false"
- MOUNT_HOST_AIRFLOW_VOLUME: "true"
FORCE_ANSWER_TO_QUESTIONS: "yes"
SKIP_CHECK_REMOTE_IMAGE: "true"
+ SKIP_CI_IMAGE_CHECK: "true"
DB_RESET: "true"
VERBOSE: "true"
- CI: "true"
- # Should be a target branch
- TRAVIS_BRANCH: "master"
- TRAVIS: "true"
+ UPGRADE_TO_LATEST_REQUIREMENTS: "false"
+ ENABLED_INTEGRATIONS: "cassandra kerberos mongo openldap presto rabbitmq
redis"
+ PYTHON_MAJOR_MINOR_VERSION: 3.6
+ # TODO: Enable cache registry once a master build succeeds uploading the
images to Github registry cache
+ DISABLE_CACHE_REGISTRY: "true"
+ CACHE_REGISTRY: "docker.pkg.github.com"
+ CACHE_IMAGE_PREFIX: ${{ github.repository }}
+ CACHE_REGISTRY_USERNAME: ${{ github.actor }}
+ CACHE_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
jobs:
- statics:
- name: Static checks
+
+ static-checks-1:
+ name: "Checks: pylint, mypy"
runs-on: ubuntu-latest
env:
- PYTHON_VERSION: 3.6
- AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
- TRAVIS_JOB_NAME: "Static"
+ MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
+ CI_JOB_TYPE: "Pylint checks for tests"
steps:
- uses: actions/checkout@master
- - name: free disk space
- run: |
- sudo swapoff -a
- sudo rm -f /swapfile
- sudo apt clean
- docker rmi $(docker image ls -aq)
- df -h
- uses: actions/setup-python@v1
with:
- python-version: ${{ env.PYTHON_VERSION }}
- - name: "Before install"
- run: ./scripts/ci/ci_before_install.sh
- - name: "Static checks"
- if: success()
+ python-version: '3.x'
+ - name: Cache pre-commit env
+ uses: actions/cache@v1
env:
- PYTHON_VERSION: 3.6
- run: ./scripts/ci/ci_run_all_static_checks.sh
+ cache-name: cache-pre-commit
+ with:
+ path: ~/.cache/pre-commit
+ key: ${{ env.cache-name }}-${{ github.job }}-${{
hashFiles('.pre-commit-config.yaml') }}
+ - name: "Free space"
+ run: ./scripts/ci/ci_free_space_on_ci.sh
+ - name: "Build CI image"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Pylint checks for tests"
+ run: |
+ python -m pip install pre-commit \
+ --constraint
requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
+ ./scripts/ci/ci_run_static_checks.sh pylint-tests mypy
- statics-tests:
- name: Pylint for tests
+ static-checks-2:
+ name: "Checks: All other"
runs-on: ubuntu-latest
env:
- PYTHON_VERSION: 3.6
- AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
- TRAVIS_JOB_NAME: "Static"
+ SKIP: pylint-tests,mypy
+ MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
+ CI_JOB_TYPE: "Static checks"
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
with:
- python-version: ${{ env.PYTHON_VERSION }}
- - name: "Before install"
- run: ./scripts/ci/ci_before_install.sh
- - name: "Static checks tests"
- if: success()
+ python-version: '3.x'
+ - name: Cache pre-commit env
+ uses: actions/cache@v1
env:
- PYTHON_VERSION: 3.6
- run: ./scripts/ci/ci_run_static_checks_pylint_tests.sh
+ cache-name: cache-pre-commit
+ with:
+ path: ~/.cache/pre-commit
+ key: ${{ env.cache-name }}-${{ github.job }}-${{
hashFiles('.pre-commit-config.yaml') }}
+ - name: "Free space"
+ run: ./scripts/ci/ci_free_space_on_ci.sh
+ - name: "Build CI image"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Static checks"
+ run: |
+ python -m pip install pre-commit \
+ --constraint
requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
+ ./scripts/ci/ci_run_static_checks.sh
docs:
name: Build docs
runs-on: ubuntu-latest
env:
- TRAVIS_JOB_NAME: "Build documentation"
- PYTHON_VERSION: 3.6
+ CI_JOB_TYPE: "Documentation"
steps:
- uses: actions/checkout@master
- - name: "Build documentation"
+ - name: "Build CI image ${{ matrix.python-version }}"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Build docs"
run: ./scripts/ci/ci_docs.sh
- tests-p36-postgres-integrations:
- name: "Tests [Postgres9.6][Py3.6][integrations]"
+ build-prod-image:
+ name: "Build prod image"
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
+ strategy:
+ matrix:
+ python-version: [3.6, 3.7]
env:
- TRAVIS_JOB_NAME: "Tests [Postgres9.6][Py3.6][integrations]"
- BACKEND: postgres
- PYTHON_VERSION: 3.6
- POSTGRES_VERSION: 9.6
- ENABLED_INTEGRATIONS: "cassandra kerberos mongo openldap rabbitmq redis"
- RUN_INTEGRATION_TESTS: all
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ CI_JOB_TYPE: "Prod image"
steps:
- uses: actions/checkout@master
- - name: free disk space
- run: |
- sudo swapoff -a
- sudo rm -f /swapfile
- sudo apt clean
- docker rmi $(docker image ls -aq)
- df -h
- - name: "Tests [Postgres9.6][Py3.6][integrations]"
- run: ./scripts/ci/ci_run_airflow_testing.sh
+ - name: "Build PROD image ${{ matrix.python-version }}"
+ run: ./scripts/ci/ci_build_production_images.sh
- tests-p36-postgres-providers:
- name: "Tests [Postgres10][Py3.6][providers]"
+ prepare-backport-packages:
+ name: "Backport packages"
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
env:
- TRAVIS_JOB_NAME: "Tests [Postgres10][Py3.6][providers]"
- BACKEND: postgres
- POSTGRES_VERSION: 10
- PYTHON_VERSION: 3.6
+ INSTALL_AIRFLOW_VERSION: "1.10.10"
+ CI_JOB_TYPE: "Prepare & test packages"
steps:
- uses: actions/checkout@master
- - name: "Tests [Postgres10][Py3.6][providers]"
- run: ./scripts/ci/ci_run_airflow_testing.sh tests/providers
+ - uses: actions/setup-python@v1
+ with:
+ python-version: '3.x'
+ - name: "Prepare & test backport packages"
+ run: |
+ python -m pip install bowler sdist wheel
+ ./scripts/ci/ci_prepare_and_test_backport_packages.sh
- tests-p36-postgres-core:
- name: "Tests [Postgres9.6][Py3.6][core]"
+ pyfiles:
+ name: "Count changed python files"
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
- env:
- TRAVIS_JOB_NAME: "Tests [Postgres9.6][Py3.6][core]"
- BACKEND: postgres
- POSTGRES_VERSION: 9.6
- PYTHON_VERSION: 3.6
+ outputs:
+ count: ${{ steps.pyfiles.outputs.count }}
steps:
- uses: actions/checkout@master
- - name: "Tests [Postgres9.6][Py3.6][core]"
- run: ./scripts/ci/ci_run_airflow_testing.sh --ignore=tests/providers
-
+ - name: "Get count of changed python files"
+ run: |
+ set +e
+ ./scripts/ci/ci_count_changed_files.sh ${GITHUB_SHA} '\.py$'
+ echo "::set-output name=count::$?"
+ id: pyfiles
- tests-p37-sqlite-integrations:
- name: "Tests [Sqlite][3.7][integrations]"
+ tests-postgres:
+ name: "Tests: Pg${{ matrix.postgres-version }}, Py{{
matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
+ needs: [static-checks-1, static-checks-2, pyfiles]
+ strategy:
+ matrix:
+ python-version: [3.6, 3.7]
+ postgres-version: [9.6, 10]
+ fail-fast: false
env:
- TRAVIS_JOB_NAME: "Tests [Sqlite][3.7][integrations]"
- BACKEND: sqlite
- PYTHON_VERSION: 3.7
- ENABLED_INTEGRATIONS: "cassandra kerberos mongo openldap rabbitmq redis"
- RUN_INTEGRATION_TESTS: all
+ BACKEND: postgres
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ POSTGRES_VERSION: ${{ matrix.postgres-version }}
+ RUN_TESTS: "true"
+ CI_JOB_TYPE: "Tests"
+ # For pull requests only run tests when python files changed
+ if: needs.pyfiles.outputs.count != '0' || github.event_name !=
'pull_request'
steps:
- uses: actions/checkout@master
- - name: free disk space
- run: |
- sudo swapoff -a
- sudo rm -f /swapfile
- sudo apt clean
- docker rmi $(docker image ls -aq)
- df -h
- - name: "Tests [Sqlite][3.7][integrations]"
+ - uses: actions/setup-python@v1
+ with:
+ python-version: '3.x'
+ - name: "Free space"
+ run: ./scripts/ci/ci_free_space_on_ci.sh
+ - name: "Build CI image ${{ matrix.python-version }}"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Tests"
run: ./scripts/ci/ci_run_airflow_testing.sh
- tests-p36-sqlite:
- name: "Tests [Sqlite][Py3.6]"
+ tests-mysql:
+ name: "Tests: MySQL${{ matrix.mysql-version }}, Py{{
matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
+ needs: [static-checks-1, static-checks-2, pyfiles]
+ strategy:
+ matrix:
+ python-version: [3.6, 3.7]
+ mysql-version: [5.7]
+ fail-fast: false
env:
- TRAVIS_JOB_NAME: "Tests [Sqlite][Py3.6]"
- BACKEND: sqlite
- PYTHON_VERSION: 3.6
+ BACKEND: mysql
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ MYSQL_VERSION: ${{ matrix.mysql-version }}
+ RUN_TESTS: "true"
+ CI_JOB_TYPE: "Tests"
+ # For pull requests only run tests when python files changed
+ if: needs.pyfiles.outputs.count != '0' || github.event_name !=
'pull_request'
steps:
- uses: actions/checkout@master
- - name: "Tests [Sqlite][Py3.6]"
+ - uses: actions/setup-python@v1
+ with:
+ python-version: '3.x'
+ - name: "Free space"
+ run: ./scripts/ci/ci_free_space_on_ci.sh
+ - name: "Build CI image ${{ matrix.python-version }}"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Tests"
run: ./scripts/ci/ci_run_airflow_testing.sh
- tests-p36-mysql-integrations:
- name: "Tests [MySQL][Py3.6][integrations]"
+ tests-sqlite:
+ name: "Tests: Sqlite Py{{ matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
+ needs: [static-checks-1, static-checks-2, pyfiles]
+ strategy:
+ matrix:
+ python-version: [3.6, 3.7]
+ fail-fast: false
env:
- TRAVIS_JOB_NAME: "Tests [MySQL][Py3.6][integrations]"
BACKEND: sqlite
- PYTHON_VERSION: 3.6
- MYSQL_VERSION: 5.7
- ENABLED_INTEGRATIONS: "cassandra kerberos mongo openldap rabbitmq redis"
- RUN_INTEGRATION_TESTS: all
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ RUN_TESTS: "true"
+ CI_JOB_TYPE: "Tests"
+ # For pull requests only run tests when python files changed
+ if: needs.pyfiles.outputs.count != '0' || github.event_name !=
'pull_request'
steps:
- uses: actions/checkout@master
- - name: free disk space
- run: |
- sudo swapoff -a
- sudo rm -f /swapfile
- sudo apt clean
- docker rmi $(docker image ls -aq)
- df -h
- - name: "Tests [MySQL][Py3.6][integrations]"
+ - uses: actions/setup-python@v1
+ with:
+ python-version: '3.x'
+ - name: "Free space"
+ run: ./scripts/ci/ci_free_space_on_ci.sh
+ - name: "Build CI image ${{ matrix.python-version }}"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Tests"
run: ./scripts/ci/ci_run_airflow_testing.sh
- tests-p36-mysql-providers:
- name: "Tests [MySQL5.7][Py3.7][providers][kerberos]"
- runs-on: ubuntu-latest
- needs: [statics, statics-tests]
- env:
- TRAVIS_JOB_NAME: "Tests [MySQL5.7][Py3.7][providers][kerberos]"
- BACKEND: mysql
- PYTHON_VERSION: 3.7
- MYSQL_VERSION: 5.7
- ENABLED_INTEGRATIONS: "kerberos"
- steps:
- - uses: actions/checkout@master
- - name: "Tests [MySQL5.7][Py3.7][providers][kerberos]"
- run: ./scripts/ci/ci_run_airflow_testing.sh tests/providers
-
- tests-p37-mysql-kerberos:
- name: "Tests [MySQL5.7][Py3.7][core][kerberos]"
- runs-on: ubuntu-latest
- needs: [statics, statics-tests]
- env:
- TRAVIS_JOB_NAME: "Tests [MySQL5.7][Py3.7][core][kerberos]"
- BACKEND: mysql
- PYTHON_VERSION: 3.7
- MYSQL_VERSION: 5.7
- ENABLED_INTEGRATIONS: "kerberos"
- steps:
- - uses: actions/checkout@master
- - name: free disk space
- run: |
- sudo swapoff -a
- sudo rm -f /swapfile
- sudo apt clean
- docker rmi $(docker image ls -aq)
- df -h
- - name: "Tests [MySQL5.7][Py3.7][core][kerberos]"
- run: ./scripts/ci/ci_run_airflow_testing.sh --ignore=tests/providers
-
requirements:
- name: "Generate requirements"
+ name: "Requirements: Py ${{ matrix.python-version}} "
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
strategy:
matrix:
python-version: [3.6, 3.7]
+ env:
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ SHOW_GENERATE_REQUIREMENTS_INSTRUCTIONS: true
+ CI_JOB_TYPE: "Requirements"
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
+ - name: "Build CI image ${{ matrix.python-version }}"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
- name: "Generate requirements"
- env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- SHOW_GENERATE_REQUIREMENTS_INSTRUCTIONS: true
run: ./scripts/ci/ci_generate_requirements.sh
- build-image:
- name: "Build production image"
+ push-images-to-github-cache:
+ name: "Build and push images to github registry"
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
+ needs:
+ - tests-sqlite
+ - tests-postgres
+ - tests-mysql
+ - requirements
+ - prepare-backport-packages
+ - build-prod-image
+ - docs
Review comment:
Yes. unfortunately we cannot do one CI image build and store it in the
registry. When you run pull requests from the forks, the GITHUB_TOKEN has only
read permissions and you are not able to push image to Github Image Registry
:(. And when you think about it, it makes perfect sense (Imagine someone
changing the build and overriding all temporary images). However I believe
(still need to verify it) that we can pull the images from the Github Registy
rather than from the DockerHub and it should significantly speed up Pullng the
images for rebuilds (it takes 3-4 minutes now). I yet have to confirm it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
