turbaszek commented on a change in pull request #8393: Bring back CI
optimisations
URL: https://github.com/apache/airflow/pull/8393#discussion_r410708187
##########
File path: .github/workflows/ci.yml
##########
@@ -17,272 +17,275 @@
#
---
-name: Airflow
+name: CI Build
on:
+ schedule:
+ - cron: '10 2 * * *'
push:
- branches: ['*']
+ branches: ['master', 'v1-10-test', 'v1-10-stable']
pull_request:
- branches: [master]
+ branches: ['master', 'v1-10-test', 'v1-10-stable']
env:
- BUILD_ID: ${{github.sha }}
MOUNT_LOCAL_SOURCES: "false"
- MOUNT_HOST_AIRFLOW_VOLUME: "true"
FORCE_ANSWER_TO_QUESTIONS: "yes"
SKIP_CHECK_REMOTE_IMAGE: "true"
+ SKIP_CI_IMAGE_CHECK: "true"
DB_RESET: "true"
VERBOSE: "true"
- CI: "true"
- # Should be a target branch
- TRAVIS_BRANCH: "master"
- TRAVIS: "true"
+ UPGRADE_TO_LATEST_REQUIREMENTS: "false"
+ ENABLED_INTEGRATIONS: "cassandra kerberos mongo openldap presto rabbitmq
redis"
+ PYTHON_MAJOR_MINOR_VERSION: 3.6
+ # TODO: Enable cache registry once a master build succeeds uploading the
images to Github registry cache
+ DISABLE_CACHE_REGISTRY: "true"
+ CACHE_REGISTRY: "docker.pkg.github.com"
+ CACHE_IMAGE_PREFIX: ${{ github.repository }}
+ CACHE_REGISTRY_USERNAME: ${{ github.actor }}
+ CACHE_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
jobs:
- statics:
- name: Static checks
+
+ static-checks-1:
+ name: "Checks: pylint, mypy"
runs-on: ubuntu-latest
env:
- PYTHON_VERSION: 3.6
- AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
- TRAVIS_JOB_NAME: "Static"
+ MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
+ CI_JOB_TYPE: "Pylint checks for tests"
steps:
- uses: actions/checkout@master
- - name: free disk space
- run: |
- sudo swapoff -a
- sudo rm -f /swapfile
- sudo apt clean
- docker rmi $(docker image ls -aq)
- df -h
- uses: actions/setup-python@v1
with:
- python-version: ${{ env.PYTHON_VERSION }}
- - name: "Before install"
- run: ./scripts/ci/ci_before_install.sh
- - name: "Static checks"
- if: success()
+ python-version: '3.x'
+ - name: Cache pre-commit env
+ uses: actions/cache@v1
env:
- PYTHON_VERSION: 3.6
- run: ./scripts/ci/ci_run_all_static_checks.sh
+ cache-name: cache-pre-commit
+ with:
+ path: ~/.cache/pre-commit
+ key: ${{ env.cache-name }}-${{ github.job }}-${{
hashFiles('.pre-commit-config.yaml') }}
+ - name: "Free space"
+ run: ./scripts/ci/ci_free_space_on_ci.sh
+ - name: "Build CI image"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Pylint checks for tests"
+ run: |
+ python -m pip install pre-commit \
+ --constraint
requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
+ ./scripts/ci/ci_run_static_checks.sh pylint-tests mypy
- statics-tests:
- name: Pylint for tests
+ static-checks-2:
+ name: "Checks: All other"
runs-on: ubuntu-latest
env:
- PYTHON_VERSION: 3.6
- AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
- TRAVIS_JOB_NAME: "Static"
+ SKIP: pylint-tests,mypy
+ MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
+ CI_JOB_TYPE: "Static checks"
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
with:
- python-version: ${{ env.PYTHON_VERSION }}
- - name: "Before install"
- run: ./scripts/ci/ci_before_install.sh
- - name: "Static checks tests"
- if: success()
+ python-version: '3.x'
+ - name: Cache pre-commit env
+ uses: actions/cache@v1
env:
- PYTHON_VERSION: 3.6
- run: ./scripts/ci/ci_run_static_checks_pylint_tests.sh
+ cache-name: cache-pre-commit
+ with:
+ path: ~/.cache/pre-commit
+ key: ${{ env.cache-name }}-${{ github.job }}-${{
hashFiles('.pre-commit-config.yaml') }}
+ - name: "Free space"
+ run: ./scripts/ci/ci_free_space_on_ci.sh
+ - name: "Build CI image"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Static checks"
+ run: |
+ python -m pip install pre-commit \
+ --constraint
requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
+ ./scripts/ci/ci_run_static_checks.sh
docs:
name: Build docs
runs-on: ubuntu-latest
env:
- TRAVIS_JOB_NAME: "Build documentation"
- PYTHON_VERSION: 3.6
+ CI_JOB_TYPE: "Documentation"
steps:
- uses: actions/checkout@master
- - name: "Build documentation"
+ - name: "Build CI image ${{ matrix.python-version }}"
+ run: ./scripts/ci/ci_prepare_image_on_ci.sh
+ - name: "Build docs"
run: ./scripts/ci/ci_docs.sh
- tests-p36-postgres-integrations:
- name: "Tests [Postgres9.6][Py3.6][integrations]"
+ build-prod-image:
+ name: "Build prod image"
runs-on: ubuntu-latest
- needs: [statics, statics-tests]
Review comment:
Is there any particular reason for skipping this? (Or is it just for faster
feedback during development?)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
