This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 34ca2f3089 Use the new breeze in CI static checks (#23187)
34ca2f3089 is described below
commit 34ca2f30899331c7330387c264ac8fb29f556a22
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Apr 23 21:33:08 2022 +0200
Use the new breeze in CI static checks (#23187)
* breeze static-checks is used in CI
* static-checks command is removed from ./breeze-legacy
* old bash pre-commit for static-checks is removed
* names are verified for existence in .pre-commit-config.yaml
* the names are used to generate documentation in STATIC_CODE_CHECKS.rst
* there is no more need to update STATIC_CODE_CHECKS.rst or
breeze-complete when new static check is added.
* The .pre-commmit-config.yaml file is a single source of truth for list
of static checks.
Fixes: #21099
---
.github/workflows/ci.yml | 8 +-
.pre-commit-config.yaml | 169 +++++----
BREEZE.rst | 14 +-
STATIC_CODE_CHECKS.rst | 396 +++++++++++----------
breeze-complete | 101 ------
breeze-legacy | 126 +------
dev/breeze/README.md | 2 +-
dev/breeze/setup.cfg | 1 +
dev/breeze/src/airflow_breeze/breeze.py | 22 +-
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 92 +++--
docs/conf.py | 3 +-
docs/exts/docs_build/fetch_inventories.py | 4 +-
images/breeze/output-static-checks.svg | 56 +--
kubernetes_tests/test_other_executors.py | 1 -
scripts/ci/libraries/_initialization.sh | 2 -
scripts/ci/pre_commit/common_precommit_utils.py | 35 ++
.../pre_commit_check_pre_commit_hook_names.py | 166 ---------
.../pre_commit_check_pre_commit_hooks.py | 186 ++++++++++
.../ci/pre_commit/pre_commit_check_pre_commits.sh | 72 ----
scripts/ci/pre_commit/pre_commit_insert_extras.py | 21 +-
.../ci/static_checks/run_basic_static_checks.sh | 51 ---
scripts/ci/static_checks/run_static_checks.sh | 51 ---
22 files changed, 642 insertions(+), 937 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1961bca289..1b71bc7669 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -609,7 +609,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
path: 'airflow/ui/node_modules'
key: ${{ runner.os }}-ui-node-modules-${{
hashFiles('airflow/ui/**/yarn.lock') }}
- name: "Static checks"
- run: ./scripts/ci/static_checks/run_static_checks.sh
+ run: breeze static-checks --all-files --show-diff-on-failure --color
always
env:
VERBOSE: false
SKIP: "identity"
@@ -625,7 +625,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
needs: [build-info]
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- SKIP: "build,mypy,flake8,identity"
+ SKIP:
"build,run-mypy,run-flake8,lint-javascript,update-migration-references,identity"
MOUNT_SELECTED_LOCAL_SOURCES: "true"
if: needs.build-info.outputs.basic-checks-only == 'true'
steps:
@@ -666,7 +666,9 @@
pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-v
${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys:
pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}
- name: "Static checks: basic checks only"
- run: ./scripts/ci/static_checks/run_basic_static_checks.sh "${{
github.sha }}"
+ run: >
+ breeze static-checks --all-files --show-diff-on-failure --color
always
+ --commit-ref "${{ github.sha }}"
env:
VERBOSE: false
docs:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 18aec9c6f2..3de0cb7cee 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -24,7 +24,9 @@ repos:
- repo: meta
hooks:
- id: identity
+ name: Print input to the static check hooks for troubleshooting
- id: check-hooks-apply
+ name: Check if all hooks apply to the repository
- repo: https://github.com/thlorenz/doctoc.git
rev: v2.1.0
hooks:
@@ -40,6 +42,7 @@ repos:
rev: v1.1.10
hooks:
- id: forbid-tabs
+ name: Fail if tabs are used in the project
exclude: ^airflow/_vendor/|^clients/gen/go\.sh$|^\.gitmodules$
- id: insert-license
name: Add license for all SQL files
@@ -51,17 +54,6 @@ repos:
- --license-filepath
- license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- - id: insert-license
- name: Add license for all other files
- exclude: ^\.github/.*$|^airflow/_vendor/
- args:
- - --comment-style
- - "|#|"
- - --license-filepath
- - license-templates/LICENSE.txt
- - --fuzzy-match-generates-todo
- files: >
-
\.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
- id: insert-license
name: Add license for all rst files
exclude: ^\.github/.*$|^airflow/_vendor/|newsfragments/.*\.rst$
@@ -143,41 +135,65 @@ repos:
- --license-filepath
- license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
+ - id: insert-license
+ name: Add license for all other files
+ exclude: ^\.github/.*$|^airflow/_vendor/
+ args:
+ - --comment-style
+ - "|#|"
+ - --license-filepath
+ - license-templates/LICENSE.txt
+ - --fuzzy-match-generates-todo
+ files: >
+
\.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
- repo: https://github.com/psf/black
rev: 22.3.0
hooks:
- id: black
+ name: Run Black (the uncompromising Python code formatter)
args: [--config=./pyproject.toml]
exclude: ^airflow/_vendor/
- repo: https://github.com/asottile/blacken-docs
rev: v1.12.1
hooks:
- id: blacken-docs
+ name: Run black on python code blocks in documentation files
alias: black
additional_dependencies: [black==22.3.0]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.1.0
hooks:
- id: check-merge-conflict
+ name: Check that merge conflicts are not being committed
- id: debug-statements
+ name: Detect accidentally committed debug statements
- id: check-builtin-literals
+ name: Require literal syntax when initializing Python builtin types
exclude: ^airflow/_vendor/
- id: detect-private-key
+ name: Detect if private key is added to the repository
- id: end-of-file-fixer
+ name: Make sure that there is an empty line at the end
exclude: ^airflow/_vendor/
- id: mixed-line-ending
+ name: Detect if mixed line ending is used (\r vs. \r\n)
exclude: ^airflow/_vendor/
- id: check-executables-have-shebangs
+ name: Check that executables have shebang
exclude: ^airflow/_vendor/
- id: check-xml
+ name: Check XML files with xmllint
exclude: ^airflow/_vendor/
- id: trailing-whitespace
+ name: Remove trailing whitespace at end of line
exclude: ^airflow/_vendor/
- id: fix-encoding-pragma
+ name: Remove encoding header from python files
exclude: ^airflow/_vendor/
args:
- --remove
- id: pretty-format-json
+ name: Format json files
args:
- --autofix
- --no-sort-keys
@@ -190,14 +206,17 @@ repos:
rev: v2.31.0
hooks:
- id: pyupgrade
+ name: Upgrade Python code automatically
args: ["--py36-plus"]
exclude: ^airflow/_vendor/
- repo: https://github.com/pre-commit/pygrep-hooks
rev: v1.9.0
hooks:
- id: rst-backticks
+ name: Check if RST files use double backticks for code
exclude: ^airflow/_vendor/
- id: python-no-log-warn
+ name: Check if there are no deprecate log warn
exclude: ^airflow/_vendor/
- repo: https://github.com/adrienverge/yamllint
rev: v1.26.3
@@ -239,6 +258,7 @@ repos:
rev: v1.3.0
hooks:
- id: yesqa
+ name: Remove unnecessary noqa statements
exclude: |
(?x)
^airflow/_vendor/
@@ -247,6 +267,7 @@ repos:
rev: '0.69'
hooks:
- id: flynt
+ name: Run flynt string format converter for Python
exclude: |
(?x)
^airflow/_vendor/
@@ -273,7 +294,7 @@ repos:
- --exclude-file=.codespellignorelines
- repo: local
hooks:
- - id: autoflake
+ - id: static-check-autoflake
name: Remove all unused code
entry: autoflake --remove-all-unused-imports
--ignore-init-module-imports --in-place
language: python
@@ -296,14 +317,14 @@ repos:
files: Dockerfile.*$
pass_filenames: true
require_serial: true
- - id: setup-order
+ - id: check-setup-order
name: Check order of dependencies in setup.cfg and setup.py
language: python
files: ^setup\.cfg$|^setup\.py$
pass_filenames: false
entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
additional_dependencies: ['rich']
- - id: setup-extra-packages
+ - id: check-extra-packages-references
name: Checks setup extra packages
description: Checks if all the libraries in setup.py are listed in
extra-packages-ref.rst file
language: python
@@ -312,7 +333,7 @@ repos:
entry:
./scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
additional_dependencies: ['rich==9.2.0']
# This check might be removed when min-airflow-version in providers is
2.2
- - id: check-2-1-compatibility
+ - id: check-airflow-2-1-compatibility
name: Check that providers are 2.1 compatible.
entry: ./scripts/ci/pre_commit/pre_commit_check_2_1_compatibility.py
language: python
@@ -338,8 +359,8 @@ repos:
language: system
files: ^setup\.cfg$
pass_filenames: false
- - id: build-providers-dependencies
- name: Build cross-dependencies for providers packages
+ - id: update-providers-dependencies
+ name: Update cross-dependencies for providers packages
entry:
./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
language: python
files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$
@@ -376,27 +397,27 @@ repos:
language: python
files: ^docs
pass_filenames: false
- - id: pydevd
+ - id: check-pydevd-left-in-code
language: pygrep
name: Check for pydevd debug statements accidentally left
entry: "pydevd.*settrace\\("
pass_filenames: true
files: \.py$
- - id: dont-use-safe-filter
+ - id: check-safe-filter-usage-in-html
language: pygrep
name: Don't use safe in templates
description: the Safe filter is error-prone, use Markup() in code
instead
entry: "\\|\\s*safe"
files: \.html$
pass_filenames: true
- - id: no-providers-in-core-examples
+ - id: check-no-providers-in-core-examples
language: pygrep
name: No providers imports in core example DAGs
description: The core example DAGs have no dependencies other than
core Airflow
entry: "^\\s*from airflow\\.providers.*"
pass_filenames: true
files: ^airflow/example_dags/.*\.py$
- - id: no-relative-imports
+ - id: check-no-relative-imports
language: pygrep
name: No relative imports
description: Airflow style is to use absolute imports only
@@ -404,7 +425,7 @@ repos:
pass_filenames: true
files: \.py$
exclude: ^tests/|^airflow/_vendor/
- - id: language-matters
+ - id: check-for-inclusive-language
language: pygrep
name: Check for language that we do not accept as community
description: Please use "deny_list" or "allow_list" instead.
@@ -421,7 +442,7 @@ repos:
^docs/apache-airflow-providers-apache-cassandra/connections/cassandra\.rst$|
^docs/apache-airflow-providers-apache-hive/commits\.rst$|
git
- - id: base-operator
+ - id: check-base-operator-usage
language: pygrep
name: Check BaseOperator[Link] core imports
description: Make sure BaseOperator[Link] is imported from
airflow.models.baseoperator in core
@@ -436,7 +457,7 @@ repos:
^airflow/sensors/.*$|
^airflow/providers/.*$|
^dev/provider_packages/.*$
- - id: base-operator
+ - id: check-base-operator-usage
language: pygrep
name: Check BaseOperator[Link] other imports
description: Make sure BaseOperator[Link] is imported from
airflow.models outside of core
@@ -446,7 +467,7 @@ repos:
(?x)
^airflow/providers/.*\.py$
exclude: ^airflow/_vendor/
- - id: provide-create-sessions
+ - id: check-provide-create-sessions-imports
language: pygrep
name: Check provide_session and create_session imports
description: provide_session and create_session should be imported
from airflow.utils.session
@@ -455,40 +476,40 @@ repos:
files: \.py$
exclude: ^airflow/_vendor/
pass_filenames: true
- - id: incorrect-use-of-LoggingMixin
+ - id: check-incorrect-use-of-LoggingMixin
language: pygrep
name: Make sure LoggingMixin is not used alone
entry: "LoggingMixin\\(\\)"
files: \.py$
exclude: ^airflow/_vendor/
pass_filenames: true
- - id: daysago-import-check
+ - id: check-daysago-import-from-utils
language: pygrep
name: Make sure days_ago is imported from airflow.utils.dates
entry: "(airflow\\.){0,1}utils\\.dates\\.days_ago"
files: \.py$
exclude: ^airflow/_vendor/
pass_filenames: true
- - id: restrict-start_date
+ - id: check-start-date-not-used-in-defaults
language: pygrep
name: "'start_date' should not be defined in default_args in
example_dags"
entry:
"default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
files: \.*example_dags.*\.py$
exclude: ^airflow/_vendor/
pass_filenames: true
- - id: check-integrations
- name: Check if integration list is aligned
+ - id: check-integrations-are-consistent
+ name: Check if integration list is consistent in various places
entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
language: system
pass_filenames: false
files: ^common/_common_values\.sh$|^breeze-complete$
- - id: check-apache-license
+ - id: check-apache-license-rat
name: Check if licenses are OK for Apache
entry: ./scripts/ci/pre_commit/pre_commit_check_license.sh
language: system
files: ^.*LICENSE.*$|^.*LICENCE.*$
pass_filenames: false
- - id: airflow-config-yaml
+ - id: check-airflow-config-yaml-consistent
name: Checks for consistency between config.yml and default_config.cfg
language: python
entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
@@ -496,59 +517,59 @@ repos:
pass_filenames: false
require_serial: true
additional_dependencies: ['pyyaml']
- - id: boring-cyborg
+ - id: check-boring-cyborg-configuration
name: Checks for Boring Cyborg configuration consistency
language: python
entry: ./scripts/ci/pre_commit/pre_commit_boring_cyborg.py
pass_filenames: false
require_serial: true
additional_dependencies: ['pyyaml', 'termcolor==1.1.0', 'wcmatch==8.2']
- - id: sort-in-the-wild
+ - id: update-in-the-wild-to-be-sorted
name: Sort INTHEWILD.md alphabetically
entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh
language: system
files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$
require_serial: true
- - id: sort-spelling-wordlist
+ - id: update-spelling-wordlist-to-be-sorted
name: Sort alphabetically and uniquify spelling_wordlist.txt
entry: ./scripts/ci/pre_commit/pre_commit_sort_spelling_wordlist.sh
language: system
files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$
require_serial: true
- - id: helm-lint
+ - id: lint-helm-chart
name: Lint Helm Chart
entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.sh
language: system
pass_filenames: false
files: ^chart
require_serial: true
- - id: shellcheck
+ - id: run-shellcheck
name: Check Shell scripts syntax correctness
language: docker_image
entry: koalaman/shellcheck:v0.7.2 -x -a
files:
^breeze-legacy$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$
exclude: ^dev/breeze/autocomplete/.*$
- - id: stylelint
+ - id: lint-css
name: stylelint
entry: "stylelint"
language: node
files: ^airflow/www/.*\.(css|scss|sass)$
# Keep dependency versions in sync w/ airflow/www/package.json
additional_dependencies: ['[email protected]',
'[email protected]']
- - id: providers-init-file
+ - id: check-providers-init-file-missing
name: Provider init file is missing
pass_filenames: false
always_run: true
entry: ./scripts/ci/pre_commit/pre_commit_check_providers_init.sh
language: system
- - id: providers-subpackages-init-file
+ - id: check-providers-subpackages-init-file-exist
name: Provider subpackage init files are there
pass_filenames: false
always_run: true
entry:
./scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py
language: python
require_serial: true
- - id: provider-yamls
+ - id: check-provider-yaml-valid
name: Validate providers.yaml files
pass_filenames: false
entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
@@ -561,23 +582,17 @@ repos:
- 'tabulate==0.8.8'
- 'jsonpath-ng==1.5.3'
- 'rich==10.9.0'
- - id: pre-commit-descriptions
- name: Check if pre-commits are described
- entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
- language: system
- files:
^\.pre-commit-config\.yaml$|^STATIC_CODE_CHECKS\.rst$|^breeze-complete$
- require_serial: true
- - id: pre-commit-hook-names
- name: Ensure hook ids are not overly long
- entry:
./scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names.py
+ - id: check-pre-commit-information-consistent
+ name: Update information about pre-commit hooks and verify ids and
names
+ entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
args:
- --max-length=70
language: python
files:
^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names\.py$
- additional_dependencies: ['pyyaml', 'jinja2', 'black==22.3.0']
+ additional_dependencies: ['pyyaml', 'jinja2', 'black==22.3.0',
'tabulate', 'rich']
require_serial: true
pass_filenames: false
- - id: airflow-providers-available
+ - id: check-airflow-providers-have-extras
name: Checks providers available when declared by extras in setup.py
language: python
entry:
./scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
@@ -585,14 +600,14 @@ repos:
pass_filenames: false
require_serial: true
additional_dependencies: ['rich']
- - id: update-breeze-config-hash
+ - id: update-breeze-readme-config-hash
name: Update Breeze README.md with config files hash
language: python
entry: ./scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py
files:
^dev/breeze/setup.*$|^dev/breeze/pyproject.toml$|^dev/breeze/README.md$
pass_filenames: false
require_serial: true
- - id: limit-breeze-dependencies
+ - id: check-breeze-top-dependencies-limited
name: Breeze should have small number of top-level dependencies
language: python
entry: ./scripts/tools/check_if_limited_dependencies.py
@@ -600,7 +615,7 @@ repos:
pass_filenames: false
require_serial: true
additional_dependencies: ['click', 'rich']
- - id: check-system-tests
+ - id: check-system-tests-present
name: Check if system tests have required segments of code
entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests.py
language: python
@@ -608,7 +623,7 @@ repos:
exclude:
^tests/system/providers/google/bigquery/example_bigquery_queries\.py$
pass_filenames: true
additional_dependencies: ['rich']
- - id: markdownlint
+ - id: lint-markdown
name: Run markdownlint
description: Checks the style of Markdown files.
entry: markdownlint
@@ -616,7 +631,7 @@ repos:
types: [markdown]
files: \.(md|mdown|markdown)$
additional_dependencies: ['markdownlint-cli']
- - id: json-schema
+ - id: lint-json-schema
name: Lint JSON Schema files with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
@@ -628,7 +643,7 @@ repos:
exclude: ^airflow/_vendor/
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1',
'requests==2.25.0']
- - id: json-schema
+ - id: lint-json-schema
name: Lint NodePort Service with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
@@ -639,7 +654,7 @@ repos:
files: ^scripts/ci/kubernetes/nodeport\.yaml$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1',
'requests==2.25.0']
- - id: json-schema
+ - id: lint-json-schema
name: Lint Docker compose files with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
@@ -650,7 +665,7 @@ repos:
files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1',
'requests==2.25.0']
- - id: json-schema
+ - id: lint-json-schema
name: Lint chart/values.schema.json file with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
@@ -662,13 +677,13 @@ repos:
files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1',
'requests==2.25.0']
- - id: vendor-k8s-json-schema
+ - id: update-vendored-in-k8s-json-schema
name: Vendor k8s definitions into values.schema.json
entry: ./scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py
language: python
files: ^chart/values\.schema\.json$
additional_dependencies: ['requests==2.25.0']
- - id: json-schema
+ - id: lint-json-schema
name: Lint chart/values.yaml file with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
@@ -681,7 +696,7 @@ repos:
files: ^chart/values\.yaml$|^chart/values\.schema\.json$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1',
'requests==2.25.0']
- - id: json-schema
+ - id: lint-json-schema
name: Lint airflow/config_templates/config.yml file with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
@@ -692,14 +707,14 @@ repos:
files: ^airflow/config_templates/config\.yml$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1',
'requests==2.25.0']
- - id: persist-credentials-disabled
+ - id: check-persist-credentials-disabled-in-github-workflows
name: Check that workflow files have persist-credentials disabled
entry: ./scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py
language: python
pass_filenames: true
files: ^\.github/workflows/.*\.yml$
additional_dependencies: ['PyYAML', 'rich']
- - id: docstring-params
+ - id: check-docstring-param-types
name: Check that docstrings do not specify param types
entry: ./scripts/ci/pre_commit/pre_commit_docstring_param_type.py
language: python
@@ -707,27 +722,27 @@ repos:
files: \.py$
exclude: ^airflow/_vendor/
additional_dependencies: ['rich']
- - id: chart-schema-lint
+ - id: lint-chart-schema
name: Lint chart/values.schema.json file
entry: ./scripts/ci/pre_commit/pre_commit_chart_schema.py
language: python
pass_filenames: false
files: ^chart/values\.schema\.json$
require_serial: true
- - id: inline-dockerfile-scripts
+ - id: update-inlined-dockerfile-scripts
name: Inline Dockerfile and Dockerfile.ci scripts
entry: ./scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py
language: python
pass_filenames: false
files: ^Dockerfile$|^Dockerfile.ci$|^scripts/docker/.*$
require_serial: true
- - id: changelog-duplicates
+ - id: check-changelog-has-no-duplicates
name: Check changelogs for duplicate entries
language: python
files: CHANGELOG\.txt$|CHANGELOG\.rst$
entry: ./scripts/ci/pre_commit/pre_commit_changelog_duplicates.py
pass_filenames: true
- - id: newsfragments
+ - id: check-newsfragments-are-valid
name: Check newsfragments are valid
language: python
files: newsfragments/.*\.rst
@@ -735,14 +750,14 @@ repos:
pass_filenames: true
## ADD MOST PRE-COMMITS ABOVE THAT LINE
# The below pre-commits are those requiring CI image to be built
- - id: mypy
+ - id: run-mypy
name: Run mypy for dev
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
files: ^dev/.*\.py$
require_serial: true
additional_dependencies: ['rich']
- - id: mypy
+ - id: run-mypy
name: Run mypy for core
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
@@ -750,14 +765,14 @@ repos:
exclude:
^provider_packages|^docs|^airflow/_vendor/|^airflow/providers|^airflow/migrations|^dev
require_serial: true
additional_dependencies: ['rich']
- - id: mypy
+ - id: run-mypy
name: Run mypy for providers
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
files: ^airflow/providers/.*\.py$
require_serial: true
additional_dependencies: ['rich']
- - id: mypy
+ - id: run-mypy
name: Run mypy for /docs/ folder
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
@@ -765,7 +780,7 @@ repos:
exclude: ^docs/rtd-deprecation
require_serial: true
additional_dependencies: ['rich']
- - id: flake8
+ - id: run-flake8
name: Run flake8
language: python
entry: ./scripts/ci/pre_commit/pre_commit_flake8.py
@@ -773,7 +788,7 @@ repos:
pass_filenames: true
exclude: ^airflow/_vendor/
additional_dependencies: ['rich']
- - id: ui-lint
+ - id: lint-javascript
name: ESLint against airflow/ui
language: python
'types_or': [javascript, tsx, ts]
@@ -781,7 +796,7 @@ repos:
entry: ./scripts/ci/pre_commit/pre_commit_ui_lint.py
pass_filenames: false
additional_dependencies: ['rich']
- - id: www-lint
+ - id: lint-javascript
name: ESLint against current UI JavaScript files
language: python
'types_or': [javascript]
@@ -789,7 +804,7 @@ repos:
entry: ./scripts/ci/pre_commit/pre_commit_www_lint.py
pass_filenames: false
additional_dependencies: ['rich']
- - id: migration-reference
+ - id: update-migration-references
name: Update migration ref doc
language: python
entry: ./scripts/ci/pre_commit/pre_commit_migration_reference.py
diff --git a/BREEZE.rst b/BREEZE.rst
index 0e3f8bf58e..243b51360b 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -456,7 +456,7 @@ Regular development tasks:
* Build documentation with ``breeze build-docs`` command
* Initialize local virtualenv with
``./scripts/tools/initialize_virtualenv.py`` command
* Cleanup breeze with ``breeze cleanup`` command
-* Run static checks with autocomplete support ``breeze static-check`` command
+* Run static checks with autocomplete support ``breeze static-checks`` command
* Run test specified with ``./breeze-legacy tests`` command
CI Image tasks:
@@ -891,14 +891,22 @@ You can also pass specific pre-commit flags for example
``--all-files`` :
The above will run mypy check for all files.
-There is also a convenience ``--last-commit`` flag that you can use to run
static check on last commit only:
+There is a convenience ``--last-commit`` flag that you can use to run static
check on last commit only:
.. code-block:: bash
breeze static-checks -t mypy --last-commit
-The above will run mypy check for all files.
+The above will run mypy check for all files in the last commit.
+
+There is another convenience ``--commit-ref`` flag that you can use to run
static check on specific commit:
+
+.. code-block:: bash
+
+ breeze static-checks -t mypy --commit-ref
639483d998ecac64d0fef7c5aa4634414065f690
+The above will run mypy check for all files in the
639483d998ecac64d0fef7c5aa4634414065f690 commit.
+Any ``commit-ish`` reference from Git will work here (branch, tag, short/long
hash etc.)
If you ever need to get a list of the files that will be checked (for
troubleshooting) use these commands:
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 00c4f55516..b0f217cfcd 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -116,190 +116,204 @@ require Breeze Docker images to be installed locally.
.. note:: Disabling particular checks
- In case you have a problem with running particular ``pre-commit`` check
you can still continue using the
- benefits of having ``pre-commit`` installed, with some of the checks
disabled. In order to disable
- checks you need to set ``SKIP`` environment variable to coma-separated
list of checks to skip. For example
- when you want to skip all checks that require Breeze Docker image to be
installed, you should be able to
- do it by setting ``export SKIP=bat-in-container-tests,build,flake8,mypy``.
You can also add this to your
- ``.bashrc`` or ``.zshrc`` if you do not want to set it manually every time
you enter the terminal.
-
-====================================
================================================================ ============
-**Checks** **Description**
**Breeze**
-====================================
================================================================ ============
-``airflow-config-yaml`` Checks that airflow config YAML is 1-1
with the code
-------------------------------------
---------------------------------------------------------------- ------------
-``airflow-providers-available`` Checks that providers are properly
declared by extras
-------------------------------------
---------------------------------------------------------------- ------------
-``airflow-provider-yaml-files-ok`` Checks that providers YAML files are
valid
-------------------------------------
---------------------------------------------------------------- ------------
-``autoflake`` Remove unused imports and unnecessary
code
-------------------------------------
---------------------------------------------------------------- ------------
-``base-operator`` Checks that BaseOperator is imported
properly
-------------------------------------
---------------------------------------------------------------- ------------
-``black`` Runs Black (the uncompromising Python
code formatter)
-------------------------------------
---------------------------------------------------------------- ------------
-``blacken-docs`` Run black on python code blocks in
documentation files
-------------------------------------
---------------------------------------------------------------- ------------
-``boring-cyborg`` Checks for Boring Cyborg configuration
consistency
-------------------------------------
---------------------------------------------------------------- ------------
-``build-providers-dependencies`` Regenerates the JSON file with
cross-provider dependencies
-------------------------------------
---------------------------------------------------------------- ------------
-``chart-schema-lint`` Lint chart/values.schema.json file
-------------------------------------
---------------------------------------------------------------- ------------
-``changelog-duplicates`` Checks for duplicate changelog entries
-------------------------------------
---------------------------------------------------------------- ------------
-``check-2-1-compatibility`` Check that providers are 2.1-compatible
-------------------------------------
---------------------------------------------------------------- ------------
-``check-apache-license`` Checks compatibility with Apache
License requirements
-------------------------------------
---------------------------------------------------------------- ------------
-``check-builtin-literals`` Require literal syntax when
initializing Python builtin types
-------------------------------------
---------------------------------------------------------------- ------------
-``check-executables-have-shebangs`` Checks that executables have shebang
-------------------------------------
---------------------------------------------------------------- ------------
-``check-extras-order`` Checks that extras in Dockerfile are
sorted
-------------------------------------
---------------------------------------------------------------- ------------
-``check-hooks-apply`` Checks which hooks are applicable to
the repository
-------------------------------------
---------------------------------------------------------------- ------------
-``check-integrations`` Checks if integration list is
synchronized in code
-------------------------------------
---------------------------------------------------------------- ------------
-``check-merge-conflicts`` Checks that merge conflicts are not
being committed
-------------------------------------------------------------------------------------------------------------------
-``check-revision-heads-map`` Checks that REVISION_HEADS_MAP is
up-to-date
-------------------------------------
---------------------------------------------------------------- ------------
-``check-xml`` Checks XML files with xmllint
-------------------------------------
---------------------------------------------------------------- ------------
-``check-system-tests`` Check if system tests have required
segments of code
-------------------------------------
---------------------------------------------------------------- ------------
-``daysago-import-check`` Checks if daysago is properly imported
-------------------------------------
---------------------------------------------------------------- ------------
-``debug-statements`` Detects accidentally committed debug
statements
-------------------------------------
---------------------------------------------------------------- ------------
-``detect-private-key`` Detects if private key is added to the
repository
-------------------------------------
---------------------------------------------------------------- ------------
-``docstring-params`` Checks that param types not specified
in docstring
-------------------------------------
---------------------------------------------------------------- ------------
-``doctoc`` Refreshes the table of contents for MD
files
-------------------------------------
---------------------------------------------------------------- ------------
-``dont-use-safe-filter`` Don't use safe in templates
-------------------------------------
---------------------------------------------------------------- ------------
-``no-providers-in-core-examples`` Don't use providers imports in core
example DAGs
-------------------------------------
---------------------------------------------------------------- ------------
-``no-relative-imports`` Use absolute imports, not relative
-------------------------------------
---------------------------------------------------------------- ------------
-``end-of-file-fixer`` Makes sure that there is an empty line
at the end.
-------------------------------------
---------------------------------------------------------------- ------------
-``fix-encoding-pragma`` Removes encoding header from python
files
-------------------------------------
---------------------------------------------------------------- ------------
-``pyupgrade`` Runs pyupgrade
-------------------------------------
---------------------------------------------------------------- ------------
-``flake8`` Runs flake8
*
-------------------------------------
---------------------------------------------------------------- ------------
-``flynt`` Runs flynt
-------------------------------------
---------------------------------------------------------------- ------------
-``codespell`` Checks for common misspellings in files.
-------------------------------------
---------------------------------------------------------------- ------------
-``forbid-tabs`` Fails if tabs are used in the project
-------------------------------------
---------------------------------------------------------------- ------------
-``helm-lint`` Verifies if helm lint passes for the
chart
-------------------------------------
---------------------------------------------------------------- ------------
-``identity`` Prints inputs to the static check hooks
for troubleshooting
-------------------------------------
---------------------------------------------------------------- ------------
-``incorrect-use-of-LoggingMixin`` Checks if LoggingMixin is properly
imported
-------------------------------------
---------------------------------------------------------------- ------------
-``inline-dockerfile-scripts`` Inline Dockerfile and Dockerfile.ci
scripts
-------------------------------------
---------------------------------------------------------------- ------------
-``insert-license`` Adds licenses for most file types
-------------------------------------
---------------------------------------------------------------- ------------
-``isort`` Sorts imports in python files
-------------------------------------
---------------------------------------------------------------- ------------
-``json-schema`` Checks that the files complies with the
JSON Schema spec
-------------------------------------
---------------------------------------------------------------- ------------
-``language-matters`` Check for language that we do not
accept as community
-------------------------------------
---------------------------------------------------------------- ------------
-``lint-dockerfile`` Lints a dockerfile
-------------------------------------
---------------------------------------------------------------- ------------
-``lint-openapi`` Lints openapi specification
-------------------------------------
---------------------------------------------------------------- ------------
-``markdownlint`` Lints Markdown files
-------------------------------------
---------------------------------------------------------------- ------------
-``migration-reference`` Update migration reference doc and
revision metadata *
-------------------------------------
---------------------------------------------------------------- ------------
-``mixed-line-ending`` Detects if mixed line ending is used
(\r vs. \r\n)
-------------------------------------
---------------------------------------------------------------- ------------
-``mypy`` Runs mypy
*
-------------------------------------
---------------------------------------------------------------- ------------
-``newsfragments`` Check newsfragments are valid - line
count by type
-------------------------------------
---------------------------------------------------------------- ------------
-``persist-credentials-disabled`` Check that workflow files have
persist-credentials disabled
-------------------------------------
---------------------------------------------------------------- ------------
-``pre-commit-descriptions`` Check if all pre-commits are described
in docs
-------------------------------------
---------------------------------------------------------------- ------------
-``pre-commit-hook-names`` Check that hook names are not overly
long
-------------------------------------
---------------------------------------------------------------- ------------
-``pretty-format-json`` Formats json files
-------------------------------------
---------------------------------------------------------------- ------------
-``provide-create-sessions`` Make sure provide-session and
create-session imports are OK
-------------------------------------
---------------------------------------------------------------- ------------
-``providers-changelogs`` Updates documentation for providers
changelogs
-------------------------------------
---------------------------------------------------------------- ------------
-``providers-subpackages-init-file`` Check that providers' subpackages
__init__.py files are there
-------------------------------------
---------------------------------------------------------------- ------------
-``providers-init-file`` Check that provider's __init__.py file
is removed
-------------------------------------
---------------------------------------------------------------- ------------
-``provider-yamls`` Checks that provider.yaml files have
the correct content
-------------------------------------
---------------------------------------------------------------- ------------
-``pydevd`` Check for accidentally committed pydevd
statements
-------------------------------------
---------------------------------------------------------------- ------------
-``pydocstyle`` Runs pydocstyle
-------------------------------------
---------------------------------------------------------------- ------------
-``python-no-log-warn`` Checks if there are no deprecate log
warn
-------------------------------------
---------------------------------------------------------------- ------------
-``restrict-start_date`` 'start_date' should not be in
default_args in example_dags
-------------------------------------
---------------------------------------------------------------- ------------
-``rst-backticks`` Checks if RST files use double
backticks for code
-------------------------------------
---------------------------------------------------------------- ------------
-``setup-order`` Checks for an order of dependencies in
setup.py
-------------------------------------
---------------------------------------------------------------- ------------
-``setup-extra-packages`` Checks if all the libraries in setup.py
are listed in docs
-------------------------------------
---------------------------------------------------------------- ------------
-``shellcheck`` Checks shell files with shellcheck
-------------------------------------
---------------------------------------------------------------- ------------
-``sort-in-the-wild`` Sort INTHEWILD.md alphabetically
-------------------------------------
---------------------------------------------------------------- ------------
-``sort-spelling-wordlist`` Sort alphabetically and uniquify
spelling_wordlist.txt
-------------------------------------
---------------------------------------------------------------- ------------
-``stylelint`` Checks CSS files with stylelint
-------------------------------------
---------------------------------------------------------------- ------------
-``trailing-whitespace`` Removes trailing whitespace at end of
line
-------------------------------------
---------------------------------------------------------------- ------------
-``ui-lint`` Static checks of airflow/ui/ folder
*
-------------------------------------
---------------------------------------------------------------- ------------
-``update-breeze-file`` Update output of breeze commands in
BREEZE.rst
-------------------------------------
---------------------------------------------------------------- ------------
-``limit-breeze-dependencies`` Breeze should have small number of
top-level dependencies
-------------------------------------
---------------------------------------------------------------- ------------
-``update-breeze-config-hash`` Update Breeze README.md with config
files hash
-------------------------------------
---------------------------------------------------------------- ------------
-``update-extras`` Updates extras in the documentation
-------------------------------------
---------------------------------------------------------------- ------------
-``update-local-yml-file`` Updates mounts in local.yml file
-------------------------------------
---------------------------------------------------------------- ------------
-``update-setup-cfg-file`` Update setup.cfg file with all licenses
-------------------------------------
---------------------------------------------------------------- ------------
-``update-supported-versions`` Updates supported versions in
documentation
-------------------------------------
---------------------------------------------------------------- ------------
-``update-versions`` Updates latest versions in the
documentation
-------------------------------------
---------------------------------------------------------------- ------------
-``vendor-k8s-json-schema`` Vendor k8s schema definitions in the
helm chart schema file
-------------------------------------
---------------------------------------------------------------- ------------
-``verify-db-migrations-documented`` Verify DB Migrations have been
documented
-------------------------------------
---------------------------------------------------------------- ------------
-``www-lint`` Static checks of js in
airflow/www/static/js/ folder *
-------------------------------------
---------------------------------------------------------------- ------------
-``yesqa`` Removes unnecessary noqa statements
-------------------------------------
---------------------------------------------------------------- ------------
-``yamllint`` Checks YAML files with yamllint
-====================================
================================================================ ============
+ In case you have a problem with running particular ``pre-commit`` check you
can still continue using the
+ benefits of having ``pre-commit`` installed, with some of the checks
disabled. In order to disable
+ checks you might need to set ``SKIP`` environment variable to coma-separated
list of checks to skip. For example
+ when you want to skip some checks (flake/mypy for example), you should be
able to do it by setting
+ ``export SKIP=run-flake8,run-mypy``. You can also add this to your
``.bashrc`` or ``.zshrc`` if you
+ do not want to set it manually every time you enter the terminal.
+
+ .. BEGIN AUTO-GENERATED STATIC CHECK LIST
+
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| ID | Description
| Image |
++========================================================+====================================================================+=========+
+| black | Run Black (the
uncompromising Python code formatter) | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| blacken-docs | Run black on python
code blocks in documentation files | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-airflow-2-1-compatibility | Check that
providers are 2.1 compatible. | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-airflow-config-yaml-consistent | Checks for
consistency between config.yml and default_config.cfg | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-airflow-providers-have-extras | Checks providers
available when declared by extras in setup.py | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-apache-license-rat | Check if licenses
are OK for Apache | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-base-operator-usage | * Check
BaseOperator[Link] core imports | |
+| | * Check
BaseOperator[Link] other imports | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-boring-cyborg-configuration | Checks for Boring
Cyborg configuration consistency | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-breeze-top-dependencies-limited | Breeze should have
small number of top-level dependencies | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-builtin-literals | Require literal
syntax when initializing Python builtin types | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-changelog-has-no-duplicates | Check changelogs
for duplicate entries | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-daysago-import-from-utils | Make sure days_ago
is imported from airflow.utils.dates | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-docstring-param-types | Check that
docstrings do not specify param types | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-executables-have-shebangs | Check that
executables have shebang | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-extra-packages-references | Checks setup extra
packages | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-extras-order | Check order of
extras in Dockerfile | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-for-inclusive-language | Check for language
that we do not accept as community | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-hooks-apply | Check if all hooks
apply to the repository | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-incorrect-use-of-LoggingMixin | Make sure
LoggingMixin is not used alone | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-integrations-are-consistent | Check if
integration list is consistent in various places | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-merge-conflict | Check that merge
conflicts are not being committed | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-newsfragments-are-valid | Check newsfragments
are valid | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-no-providers-in-core-examples | No providers
imports in core example DAGs | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-no-relative-imports | No relative imports
| |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-persist-credentials-disabled-in-github-workflows | Check that workflow
files have persist-credentials disabled | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-pre-commit-information-consistent | Update information
about pre-commit hooks and verify ids and names | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-provide-create-sessions-imports | Check
provide_session and create_session imports | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-provider-yaml-valid | Validate
providers.yaml files | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-providers-init-file-missing | Provider init file
is missing | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-providers-subpackages-init-file-exist | Provider subpackage
init files are there | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-pydevd-left-in-code | Check for pydevd
debug statements accidentally left | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-revision-heads-map | Check that the
REVISION_HEADS_MAP is up-to-date | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-safe-filter-usage-in-html | Don't use safe in
templates | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-setup-order | Check order of
dependencies in setup.cfg and setup.py | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-start-date-not-used-in-defaults | 'start_date' should
not be defined in default_args in example_dags | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-system-tests-present | Check if system
tests have required segments of code | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| check-xml | Check XML files
with xmllint | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| codespell | Run codespell to
check for common misspellings in files | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| debug-statements | Detect accidentally
committed debug statements | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| detect-private-key | Detect if private
key is added to the repository | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| doctoc | Add TOC for md and
rst files | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| end-of-file-fixer | Make sure that
there is an empty line at the end | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| fix-encoding-pragma | Remove encoding
header from python files | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| flynt | Run flynt string
format converter for Python | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| forbid-tabs | Fail if tabs are
used in the project | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| identity | Print input to the
static check hooks for troubleshooting | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| insert-license | * Add license for
all SQL files | |
+| | * Add license for
all rst files | |
+| | * Add license for
all CSS/JS/PUML/TS/TSX files | |
+| | * Add license for
all JINJA template files | |
+| | * Add license for
all shell files | |
+| | * Add license for
all Python files | |
+| | * Add license for
all XML files | |
+| | * Add license for
all YAML files | |
+| | * Add license for
all md files | |
+| | * Add license for
all other files | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| isort | Run isort to sort
imports in Python files | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| lint-chart-schema | Lint
chart/values.schema.json file | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| lint-css | stylelint
| |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| lint-dockerfile | Lint dockerfile
| |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| lint-helm-chart | Lint Helm Chart
| |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| lint-javascript | * ESLint against
airflow/ui | |
+| | * ESLint against
current UI JavaScript files | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| lint-json-schema | * Lint JSON Schema
files with JSON Schema | |
+| | * Lint NodePort
Service with JSON Schema | |
+| | * Lint Docker
compose files with JSON Schema | |
+| | * Lint
chart/values.schema.json file with JSON Schema | |
+| | * Lint
chart/values.yaml file with JSON Schema | |
+| | * Lint
airflow/config_templates/config.yml file with JSON Schema | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| lint-markdown | Run markdownlint
| |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| lint-openapi | * Lint OpenAPI
using spectral | |
+| | * Lint OpenAPI
using openapi-spec-validator | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| mixed-line-ending | Detect if mixed
line ending is used (\r vs. \r\n) | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| pretty-format-json | Format json files
| |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| pydocstyle | Run pydocstyle
| |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| python-no-log-warn | Check if there are
no deprecate log warn | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| pyupgrade | Upgrade Python code
automatically | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| rst-backticks | Check if RST files
use double backticks for code | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| run-flake8 | Run flake8
| |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| run-mypy | * Run mypy for dev
| |
+| | * Run mypy for core
| |
+| | * Run mypy for
providers | |
+| | * Run mypy for
/docs/ folder | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| run-shellcheck | Check Shell scripts
syntax correctness | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| static-check-autoflake | Remove all unused
code | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| trailing-whitespace | Remove trailing
whitespace at end of line | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-breeze-file | Update output of
breeze commands in BREEZE.rst | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-breeze-readme-config-hash | Update Breeze
README.md with config files hash | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-extras | Update extras in
documentation | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-in-the-wild-to-be-sorted | Sort INTHEWILD.md
alphabetically | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-inlined-dockerfile-scripts | Inline Dockerfile
and Dockerfile.ci scripts | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-local-yml-file | Update mounts in
the local yml file | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-migration-references | Update migration
ref doc | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-providers-dependencies | Update
cross-dependencies for providers packages | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-setup-cfg-file | Update setup.cfg
file with all licenses | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-spelling-wordlist-to-be-sorted | Sort alphabetically
and uniquify spelling_wordlist.txt | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-supported-versions | Updates supported
versions in documentation | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-vendored-in-k8s-json-schema | Vendor k8s
definitions into values.schema.json | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| update-version | Update version to
the latest version in the documentation | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| yamllint | Check YAML files
with yamllint | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+| yesqa | Remove unnecessary
noqa statements | |
++--------------------------------------------------------+--------------------------------------------------------------------+---------+
+
+ .. END AUTO-GENERATED STATIC CHECK LIST
Using pre-commit
................
@@ -317,13 +331,13 @@ code. But you can run pre-commit hooks manually as needed.
.. code-block:: bash
- pre-commit run mypy
+ pre-commit run run-mypy
- Run only mypy checks on all files by using:
.. code-block:: bash
- pre-commit run mypy --all-files
+ pre-commit run run-mypy --all-files
- Run all checks on all files by using:
@@ -351,7 +365,7 @@ code. But you can run pre-commit hooks manually as needed.
.. code-block:: bash
- SKIP=mypy,flake8,build pre-commit run --all-files
+ SKIP=run-mypy,run-flake8 pre-commit run --all-files
You can always skip running the tests by providing ``--no-verify`` flag to the
@@ -373,25 +387,25 @@ Run the ``mypy`` check for the currently staged changes:
.. code-block:: bash
- breeze static-check --type mypy
+ breeze static-check --type run-mypy
Run the ``mypy`` check for all files:
.. code-block:: bash
- breeze static-check --type mypy --all-files
+ breeze static-check --type run-mypy --all-files
Run the ``flake8`` check for the ``tests.core.py`` file with verbose output:
.. code-block:: bash
- breeze static-check --type flake8 --files tests/core.py --verbose
+ breeze static-check --type run-flake8 --files tests/core.py --verbose
Run the ``flake8`` check for the ``tests.core`` package with verbose output:
.. code-block:: bash
- breeze static-check --type flake8 --files tests/core/* --verbose
+ breeze static-check --type run-flake8 --files tests/core/* --verbose
Run all tests for the currently staged files:
diff --git a/breeze-complete b/breeze-complete
index 173ce24029..45a23c4e41 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -60,103 +60,6 @@ none
EOF
)
-_breeze_allowed_static_checks=$(cat <<-EOF
-all
-airflow-config-yaml
-airflow-providers-available
-airflow-provider-yaml-files-ok
-autoflake
-base-operator
-black
-blacken-docs
-boring-cyborg
-build
-build-providers-dependencies
-chart-schema-lint
-capitalized-breeze
-changelog-duplicates
-check-2-1-compatibility
-check-apache-license
-check-builtin-literals
-check-executables-have-shebangs
-check-extras-order
-check-hooks-apply
-check-integrations
-check-merge-conflict
-check-revision-heads-map
-check-xml
-check-system-tests
-daysago-import-check
-debug-statements
-detect-private-key
-docstring-params
-doctoc
-dont-use-safe-filter
-end-of-file-fixer
-fix-encoding-pragma
-flake8
-flynt
-codespell
-forbid-tabs
-helm-lint
-identity
-incorrect-use-of-LoggingMixin
-insert-license
-inline-dockerfile-scripts
-isort
-json-schema
-language-matters
-lint-dockerfile
-lint-openapi
-markdownlint
-mermaid
-migration-reference
-mixed-line-ending
-mypy
-mypy-helm
-newsfragments
-no-providers-in-core-examples
-no-relative-imports
-persist-credentials-disabled
-pre-commit-descriptions
-pre-commit-hook-names
-pretty-format-json
-provide-create-sessions
-providers-changelogs
-providers-init-file
-providers-subpackages-init-file
-provider-yamls
-pydevd
-pydocstyle
-python-no-log-warn
-pyupgrade
-restrict-start_date
-rst-backticks
-setup-order
-setup-extra-packages
-shellcheck
-sort-in-the-wild
-sort-spelling-wordlist
-stylelint
-trailing-whitespace
-ui-lint
-limit-breeze-dependencies
-update-breeze-file
-update-breeze-config-hash
-update-extras
-update-local-yml-file
-update-setup-cfg-file
-update-supported-versions
-update-versions
-vendor-k8s-json-schema
-verify-db-migrations-documented
-version-sync
-www-lint
-yamllint
-yesqa
-EOF
-)
-
_breeze_default_github_repository="apache/airflow"
_breeze_default_github_image_id="latest"
@@ -196,7 +99,6 @@ toggle-suppress-asciiart"
_breeze_extra_arg_commands="
docker-compose
kind-cluster
-static-check
tests
"
@@ -249,9 +151,6 @@ function breeze_complete::get_known_values_breeze() {
--helm-version)
_breeze_known_values=${_breeze_allowed_helm_versions}
;;
- static-check)
- _breeze_known_values=${_breeze_allowed_static_checks}
- ;;
--use-airflow-version)
_breeze_known_values=${_breeze_allowed_USE_AIRFLOW_VERSION}
;;
diff --git a/breeze-legacy b/breeze-legacy
index 362e327d19..3613d6b7d8 100755
--- a/breeze-legacy
+++ b/breeze-legacy
@@ -51,10 +51,6 @@ export INTEGRATIONS
declare -a REMAINING_ARGS
export REMAINING_ARGS
-# This is where static check options are defined
-declare -a EXTRA_STATIC_CHECK_OPTIONS
-export EXTRA_STATIC_CHECK_OPTIONS
-
#######################################################################################################
# Sets up all the default variables for Breeze. They are needed by all other
functions
# All those variables are exported. They are not set to read-only because those
@@ -1097,32 +1093,6 @@ function breeze::parse_arguments() {
command_to_run="perform_setup_autocomplete"
shift
;;
- static-check)
- last_subcommand="${1}"
- command_to_run="perform_static_checks"
- if [[ "$#" -lt 2 ]]; then
- if [[ ${run_help} != "true" ]]; then
- echo "You should specify static check that you would like
to run or 'all' to run all checks."
- echo
- echo "One of :"
- echo
- echo "${_breeze_allowed_static_checks:=}"
- echo
- echo "For example:"
- echo
- echo "${CMDNAME} static-check mypy"
- echo
- exit 1
- else
- shift
- fi
- else
- export
PYTHON_MAJOR_MINOR_VERSION=${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}
- static_check="${2:-}"
- EXTRA_STATIC_CHECK_OPTIONS+=("--show-diff-on-failure")
- shift 2
- fi
- ;;
start-airflow)
last_subcommand="${1}"
export START_AIRFLOW="true"
@@ -1265,10 +1235,6 @@ function breeze::prepare_formatted_versions() {
fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
readonly FORMATTED_BACKENDS
- FORMATTED_STATIC_CHECKS=$(echo "${_breeze_allowed_static_checks=""}" | tr
'\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_STATIC_CHECKS
-
FORMATTED_INTEGRATIONS=$(echo "${_breeze_allowed_integrations=""}" | tr
'\n' ' ' |
fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
readonly FORMATTED_INTEGRATIONS
@@ -1370,8 +1336,6 @@ function breeze::prepare_usage() {
readonly USAGE_STOP
export USAGE_RESTART="Stops the docker-compose environment including DB
cleanup"
readonly USAGE_RESTART
- export USAGE_STATIC_CHECK="Performs selected static check for changed
files"
- readonly USAGE_STATIC_CHECK
export USAGE_TOGGLE_SUPPRESS_CHEATSHEET="Toggles on/off cheatsheet"
readonly USAGE_TOGGLE_SUPPRESS_CHEATSHEET
export USAGE_TOGGLE_SUPPRESS_ASCIIART="Toggles on/off asciiart"
@@ -1516,35 +1480,6 @@ Flags:
$(breeze::flag_stop_airflow)
"
readonly DETAILED_USAGE_RESTART
- export DETAILED_USAGE_STATIC_CHECK="
-${CMDNAME} static-check [FLAGS] static_check [-- <EXTRA_ARGS>]
-
- Run selected static checks for currently changed files. You should
specify static check that
- you would like to run or 'all' to run all checks. One of:
-
-${FORMATTED_STATIC_CHECKS}
-
- You can pass extra arguments including options to the pre-commit
framework as
- <EXTRA_ARGS> passed after --. For example:
-
- '${CMDNAME} static-check mypy' or
- '${CMDNAME} static-check mypy -- --files tests/core.py'
- '${CMDNAME} static-check mypy -- --all-files'
-
- To check all files that differ between you current branch and main run:
-
- '${CMDNAME} static-check all -- --from-ref \$(git merge-base main HEAD)
--to-ref HEAD'
-
- To check all files that are in the HEAD commit run:
-
- '${CMDNAME} static-check mypy -- --from-ref HEAD^ --to-ref HEAD'
-
-
- You can see all the options by adding --help EXTRA_ARG:
-
- '${CMDNAME} static-check mypy -- --help'
-"
- readonly DETAILED_USAGE_STATIC_CHECK
export DETAILED_USAGE_TESTS="
${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>]
@@ -2395,60 +2330,6 @@ function breeze::print_setup_instructions() {
fi
}
-#######################################################################################################
-#
-# Checks that pre-commit is installed and upgrades it if needed
-# this is used in case static check command is used.
-#
-#######################################################################################################
-function breeze::make_sure_precommit_is_installed() {
- echo
- echo "Making sure pre-commit is installed"
- echo
- local pip_binary
- if command -v pip3 >/dev/null; then
- pip_binary=pip3
- elif command -v pip >/dev/null; then
- pip_binary=pip
- else
- echo
- echo "${COLOR_RED}ERROR: You need to have pip or pip3 in your PATH
${COLOR_RESET}"
- echo
- exit 1
- fi
- "${pip_binary}" install --upgrade pre-commit >/dev/null 2>&1
- # Add ~/.local/bin to the path in case pip is run outside of virtualenv
- export PATH="${PATH}":~/.local/bin
-}
-
-#######################################################################################################
-#
-# Runs chosen static checks
-#
-# Uses variables:
-# static_check
-#
-#######################################################################################################
-function breeze::run_static_checks() {
- if [[ ${static_check} == "all" ]]; then
- echo
- echo "Running: pre-commit run" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
- echo
- pre-commit run "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
- else
- if [[ ${static_check} == *"mypy"* || ${static_check} == *"flake8"* ]];
then
- echo
- echo "Running build pre-commit before running the requested static
check"
- echo
- pre-commit run "build"
- fi
- echo
- echo "Running: pre-commit run" "${static_check}"
"${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
- echo
- pre-commit run "${static_check}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}"
"$@"
- fi
-}
-
#######################################################################################################
#
# Runs Build before a command if needed - the right build command will be
determined and used
@@ -2469,7 +2350,7 @@ function breeze::run_build_command() {
;;
enter_breeze)
;;
- build_docs | perform_static_checks )
+ build_docs )
;;
perform_initialize_local_virtualenv | perform_setup_autocomplete | \
toggle_suppress_cheatsheet | toggle_suppress_asciiart )
@@ -2611,11 +2492,6 @@ function breeze::run_breeze_command() {
${run_command} "${dc_run_file}" "${docker_compose_command}"
"${EXTRA_DC_OPTIONS[@]}" "$@"
set -u
;;
- perform_static_checks)
- docker_engine_resources::check_all_resources
- breeze::make_sure_precommit_is_installed
- breeze::run_static_checks "${@}"
- ;;
perform_initialize_local_virtualenv)
breeze::initialize_virtualenv
;;
diff --git a/dev/breeze/README.md b/dev/breeze/README.md
index 4f5da05561..604fdb1bf8 100644
--- a/dev/breeze/README.md
+++ b/dev/breeze/README.md
@@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY
UPDATED BY PRE-COMMIT.
---------------------------------------------------------------------------------------------------------
-Package config hash:
352a3150dd5b8763de98e99e6dde1271cd35c98df506e091008f50f4c7765ea68eb23005e26af8bc61116bd90777a3c31afd12041890c6df7fd8b518500c08ae
+Package config hash:
ea8611f8d231310d9c8dad11e7f61b5d8dd9f995473451a0ae60856e51d2264a6c1dc0a58be8fdc665673945197018b41d6b4b5aad1fdffcd011aa10c25790bc
---------------------------------------------------------------------------------------------------------
diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg
index abdc8b9c41..79e4d9a3f4 100644
--- a/dev/breeze/setup.cfg
+++ b/dev/breeze/setup.cfg
@@ -57,6 +57,7 @@ install_requires =
inputimeout
importlib-metadata>=4.4; python_version < "3.8"
pendulum
+ pre-commit
psutil
pytest
pytest-xdist
diff --git a/dev/breeze/src/airflow_breeze/breeze.py
b/dev/breeze/src/airflow_breeze/breeze.py
index cb8d6c2023..5e28860649 100755
--- a/dev/breeze/src/airflow_breeze/breeze.py
+++ b/dev/breeze/src/airflow_breeze/breeze.py
@@ -631,7 +631,7 @@ option_mount_sources = click.option(
)
option_force_build = click.option(
- '-f', '--force-build', help="Force image build no matter if it is "
"determined as needed.", is_flag=True
+ '-f', '--force-build', help="Force image build no matter if it is
determined as needed.", is_flag=True
)
option_db_reset = click.option(
@@ -1800,7 +1800,19 @@ def build_docs(
@click.option(
'-s', '--show-diff-on-failure', help="Show diff for files modified by the
checks.", is_flag=True
)
[email protected]('-c', '--last-commit', help="Run checks for all files in last
commit.", is_flag=True)
[email protected](
+ '-c',
+ '--last-commit',
+ help="Run checks for all files in last commit. Mutually exclusive with
--commit-ref.",
+ is_flag=True,
+)
[email protected](
+ '-r',
+ '--commit-ref',
+ help="Run checks for this commit reference only "
+ "(can be any git commit-ish reference). "
+ "Mutually exclusive with --last-commit.",
+)
@option_verbose
@option_dry_run
@option_github_repository
@@ -1812,12 +1824,16 @@ def static_checks(
all_files: bool,
show_diff_on_failure: bool,
last_commit: bool,
+ commit_ref: str,
type: Tuple[str],
files: bool,
precommit_args: Tuple,
):
if check_pre_commit_installed(verbose=verbose):
command_to_execute = ['pre-commit', 'run']
+ if last_commit and commit_ref:
+ console.print("\n[red]You cannot specify both --last-commit and
--commit-ref[/]\n")
+ sys.exit(1)
for single_check in type:
command_to_execute.append(single_check)
if all_files:
@@ -1826,6 +1842,8 @@ def static_checks(
command_to_execute.append("--show-diff-on-failure")
if last_commit:
command_to_execute.extend(["--from-ref", "HEAD^", "--to-ref",
"HEAD"])
+ if commit_ref:
+ command_to_execute.extend(["--from-ref", f"{commit_ref}^",
"--to-ref", f"{commit_ref}"])
if files:
command_to_execute.append("--files")
if verbose or dry_run:
diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
index 49f879bfda..40ff16a7a0 100644
--- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
+++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
@@ -22,89 +22,87 @@
# `pre_commit_ids_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze`
DIRECTORY
PRE_COMMIT_LIST = [
- 'airflow-config-yaml',
- 'airflow-providers-available',
'all',
- 'autoflake',
- 'base-operator',
'black',
'blacken-docs',
- 'boring-cyborg',
- 'build-providers-dependencies',
- 'changelog-duplicates',
- 'chart-schema-lint',
- 'check-2-1-compatibility',
- 'check-apache-license',
+ 'check-airflow-2-1-compatibility',
+ 'check-airflow-config-yaml-consistent',
+ 'check-airflow-providers-have-extras',
+ 'check-apache-license-rat',
+ 'check-base-operator-usage',
+ 'check-boring-cyborg-configuration',
+ 'check-breeze-top-dependencies-limited',
'check-builtin-literals',
+ 'check-changelog-has-no-duplicates',
+ 'check-daysago-import-from-utils',
+ 'check-docstring-param-types',
'check-executables-have-shebangs',
+ 'check-extra-packages-references',
'check-extras-order',
+ 'check-for-inclusive-language',
'check-hooks-apply',
- 'check-integrations',
+ 'check-incorrect-use-of-LoggingMixin',
+ 'check-integrations-are-consistent',
'check-merge-conflict',
+ 'check-newsfragments-are-valid',
+ 'check-no-providers-in-core-examples',
+ 'check-no-relative-imports',
+ 'check-persist-credentials-disabled-in-github-workflows',
+ 'check-pre-commit-information-consistent',
+ 'check-provide-create-sessions-imports',
+ 'check-provider-yaml-valid',
+ 'check-providers-init-file-missing',
+ 'check-providers-subpackages-init-file-exist',
+ 'check-pydevd-left-in-code',
'check-revision-heads-map',
- 'check-system-tests',
+ 'check-safe-filter-usage-in-html',
+ 'check-setup-order',
+ 'check-start-date-not-used-in-defaults',
+ 'check-system-tests-present',
'check-xml',
'codespell',
- 'daysago-import-check',
'debug-statements',
'detect-private-key',
- 'docstring-params',
'doctoc',
- 'dont-use-safe-filter',
'end-of-file-fixer',
'fix-encoding-pragma',
- 'flake8',
'flynt',
'forbid-tabs',
- 'helm-lint',
'identity',
- 'incorrect-use-of-LoggingMixin',
- 'inline-dockerfile-scripts',
'insert-license',
'isort',
- 'json-schema',
- 'language-matters',
- 'limit-breeze-dependencies',
+ 'lint-chart-schema',
+ 'lint-css',
'lint-dockerfile',
+ 'lint-helm-chart',
+ 'lint-javascript',
+ 'lint-json-schema',
+ 'lint-markdown',
'lint-openapi',
- 'markdownlint',
- 'migration-reference',
'mixed-line-ending',
- 'mypy',
- 'newsfragments',
- 'no-providers-in-core-examples',
- 'no-relative-imports',
- 'persist-credentials-disabled',
- 'pre-commit-descriptions',
- 'pre-commit-hook-names',
'pretty-format-json',
- 'provide-create-sessions',
- 'provider-yamls',
- 'providers-init-file',
- 'providers-subpackages-init-file',
- 'pydevd',
'pydocstyle',
'python-no-log-warn',
'pyupgrade',
- 'restrict-start_date',
'rst-backticks',
- 'setup-extra-packages',
- 'setup-order',
- 'shellcheck',
- 'sort-in-the-wild',
- 'sort-spelling-wordlist',
- 'stylelint',
+ 'run-flake8',
+ 'run-mypy',
+ 'run-shellcheck',
+ 'static-check-autoflake',
'trailing-whitespace',
- 'ui-lint',
- 'update-breeze-config-hash',
'update-breeze-file',
+ 'update-breeze-readme-config-hash',
'update-extras',
+ 'update-in-the-wild-to-be-sorted',
+ 'update-inlined-dockerfile-scripts',
'update-local-yml-file',
+ 'update-migration-references',
+ 'update-providers-dependencies',
'update-setup-cfg-file',
+ 'update-spelling-wordlist-to-be-sorted',
'update-supported-versions',
+ 'update-vendored-in-k8s-json-schema',
'update-version',
- 'vendor-k8s-json-schema',
- 'www-lint',
'yamllint',
'yesqa',
]
diff --git a/docs/conf.py b/docs/conf.py
index b1742f0c00..ef889f641a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -44,9 +44,10 @@ try:
except ImportError:
from yaml import SafeLoader # type: ignore[misc]
+from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES
+
import airflow
from airflow.configuration import AirflowConfigParser, default_config_yaml
-from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES
sys.path.append(os.path.join(os.path.dirname(__file__), 'exts'))
diff --git a/docs/exts/docs_build/fetch_inventories.py
b/docs/exts/docs_build/fetch_inventories.py
index d55e28ae14..0833b6de46 100644
--- a/docs/exts/docs_build/fetch_inventories.py
+++ b/docs/exts/docs_build/fetch_inventories.py
@@ -27,11 +27,11 @@ from typing import Iterator, List, Tuple
import requests
import urllib3.exceptions
+from docs.exts.docs_build.docs_builder import get_available_providers_packages
+from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES
from requests.adapters import DEFAULT_POOLSIZE
from airflow.utils.helpers import partition
-from docs.exts.docs_build.docs_builder import get_available_providers_packages
-from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES
CURRENT_DIR = os.path.dirname(__file__)
ROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir,
os.pardir))
diff --git a/images/breeze/output-static-checks.svg
b/images/breeze/output-static-checks.svg
index e63056e358..bde9d4473e 100644
--- a/images/breeze/output-static-checks.svg
+++ b/images/breeze/output-static-checks.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1242" viewBox="0 0 1720.0 1242"
+<svg width="1720.0" height="1462" viewBox="0 0 1720.0 1462"
xmlns="http://www.w3.org/2000/svg";>
<style>
@font-face {
@@ -122,34 +122,44 @@
<div><span class="r1">
</span></div>
<div><span class="r4">╭─ Pre-commit flags
───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--type</span><span class="r1"> </span><span
class="r6">-t</span><span class="r1"> </span><span class="r1">Type(s) of the
static checks to run (multiple can be added).
</span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">(airflow-config-yaml | airflow-providers-available |
all | autoflake | base-operator |</span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">black | blacken-docs | boring-cyborg |
build-providers-dependencies | </span><span class="r1">
</span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">changelog-duplicates | chart-schema-lint |
check-2-1-compatibility | </span><span class="r1">
</span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-apache-license | check-builtin-literals |
check-executables-have-shebangs | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-extras-order | check-hooks-apply |
check-integrations | check-merge-conflict | </span><span class="r1">
</span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-revision-heads-map | check-system-tests |
check-xml | codespell | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">daysago-import-check | debug-statements |
detect-private-key | docstring-params | </span><span class="r1">
</span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">doctoc | dont-use-safe-filter | end-of-file-fixer |
fix-encoding-pragma | flake8 | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">flynt | forbid-tabs | helm-lint | identity |
incorrect-use-of-LoggingMixin | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">inline-dockerfile-scripts | insert-license | isort |
json-schema | language-matters | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">limit-breeze-dependencies | lint-dockerfile |
lint-openapi | markdownlint | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">migration-reference | mixed-line-ending | mypy |
newsfragments | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">no-providers-in-core-examples | no-relative-imports |
persist-credentials-disabled | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">pre-commit-descriptions | pre-commit-hook-names |
pretty-format-json | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">provide-create-sessions | provider-yamls |
providers-init-file | </span><span class="r1">
</span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">providers-subpackages-init-file | pydevd | pydocstyle |
python-no-log-warn | pyupgrade</span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">| restrict-start_date | rst-backticks |
setup-extra-packages | setup-order | </span><span class="r1">
</span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">shellcheck | sort-in-the-wild | sort-spelling-wordlist
| stylelint | </span><span class="r1"> </span><span
class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">trailing-whitespace | ui-lint |
update-breeze-config-hash | update-breeze-file | </span><span class="r1">
</span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">update-extras | update-local-yml-file |
update-setup-cfg-file | </span><span class="r1">
</span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1">
</span><span class="r7">update-supported-versions | update-version |
vendor-k8s-json-schema | www-lint | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">(all | black | blacken-docs |
check-airflow-2-1-compatibility | </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-airflow-config-yaml-consistent |
check-airflow-providers-have-extras | </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-apache-license-rat | check-base-operator-usage |
</span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-boring-cyborg-configuration |
check-breeze-top-dependencies-limited | </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-builtin-literals |
check-changelog-has-no-duplicates | </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-daysago-import-from-utils |
check-docstring-param-types | </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-executables-have-shebangs |
check-extra-packages-references | check-extras-order</span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">| check-for-inclusive-language | check-hooks-apply |
</span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-incorrect-use-of-LoggingMixin |
check-integrations-are-consistent | </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-merge-conflict | check-newsfragments-are-valid |
</span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-no-providers-in-core-examples |
check-no-relative-imports | </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-persist-credentials-disabled-in-github-workflows
| </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-pre-commit-information-consistent |
check-provide-create-sessions-imports | </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-provider-yaml-valid |
check-providers-init-file-missing | </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-providers-subpackages-init-file-exist |
check-pydevd-left-in-code | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-revision-heads-map |
check-safe-filter-usage-in-html | check-setup-order | </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">check-start-date-not-used-in-defaults |
check-system-tests-present | check-xml | </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">codespell | debug-statements | detect-private-key |
doctoc | end-of-file-fixer | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">fix-encoding-pragma | flynt | forbid-tabs | identity |
insert-license | isort | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">lint-chart-schema | lint-css | lint-dockerfile |
lint-helm-chart | lint-javascript | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">lint-json-schema | lint-markdown | lint-openapi |
mixed-line-ending | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">pretty-format-json | pydocstyle | python-no-log-warn |
pyupgrade | rst-backticks | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">run-flake8 | run-mypy | run-shellcheck |
static-check-autoflake | trailing-whitespace </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">| update-breeze-file | update-breeze-readme-config-hash
| update-extras | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">update-in-the-wild-to-be-sorted |
update-inlined-dockerfile-scripts | </span><span class="r1">
</span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">update-local-yml-file | update-migration-references |
update-providers-dependencies | </span><span class="r1"> </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">update-setup-cfg-file |
update-spelling-wordlist-to-be-sorted | </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">update-supported-versions |
update-vendored-in-k8s-json-schema | update-version | </span><span
class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1">
</span><span class="r7">yamllint | yesqa)
</span><span class="r1"> </span><span
class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--files</span><span class="r1"> </span><span
class="r6">-f</span><span class="r1"> </span><span class="r1">List of files to
run the checks on.</span><span class="r1"> </span><span
class="r7">(TEXT)</span><span class="r1">
</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--all-files</span><span class="r1"> </span><span
class="r6">-a</span><span class="r1"> </span><span class="r1">Run checks on
all files.
</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--show-diff-on-failure</span><span class="r1"> </span><span
class="r6">-s</span><span class="r1"> </span><span class="r1">Show diff for
files modified by the checks.
</span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--last-commit</span><span class="r1"> </span><span
class="r6">-c</span><span class="r1"> </span><span class="r1">Run checks for
all files in last commit.
</span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--last-commit</span><span class="r1"> </span><span
class="r6">-c</span><span class="r1"> Run checks for all files in last commit.
Mutually exclusive with </span><span class="r5">--commit-ref</span><span
class="r1">.</span><span class="r1"> </span><span class="r1">
</span><span class="r4">│</span></div>
<div><span
class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options
────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--commit-ref</span><span class="r1"> </span><span
class="r6">-r</span><span class="r1"> Run checks for this commit reference
only (can be any git commit-ish reference). Mutually </span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
exclusive with </span><span class="r5">--last-commit</span><span class="r1">.
</span><span
class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r7">(TEXT)
</span><span class="r1"> </span><span
class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--verbose</span><span class="r1"> </span><span
class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose
information about performed steps.
</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--dry-run</span><span class="r1"> </span><span
class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is
set, commands are only printed, not executed.
</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--github-repository</span><span class="r1"> </span><span
class="r6">-g</span><span class="r1"> </span><span class="r1">GitHub
repository used to pull, push run images.</span><span class="r1"> </span><span
class="r7">(TEXT)</span><span class="r1"> </span><span class="r4">[default:
apache/airflow]</span><span class="r1"> </span><span class="r1">
</span><span class="r4">│</span></div>
diff --git a/kubernetes_tests/test_other_executors.py
b/kubernetes_tests/test_other_executors.py
index 9c92dbca4a..e2b7037779 100644
--- a/kubernetes_tests/test_other_executors.py
+++ b/kubernetes_tests/test_other_executors.py
@@ -18,7 +18,6 @@
import time
import pytest
-
from kubernetes_tests.test_base import EXECUTOR, TestBase
diff --git a/scripts/ci/libraries/_initialization.sh
b/scripts/ci/libraries/_initialization.sh
index c36d3a50a8..644b3e0fed 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -891,8 +891,6 @@ function initialization::make_constants_read_only() {
readonly FORWARD_CREDENTIALS
- readonly EXTRA_STATIC_CHECK_OPTIONS
-
readonly PYTHON_BASE_IMAGE
readonly AIRFLOW_IMAGE_KUBERNETES
readonly BUILT_CI_IMAGE_FLAG_FILE
diff --git a/scripts/ci/pre_commit/common_precommit_utils.py
b/scripts/ci/pre_commit/common_precommit_utils.py
new file mode 100644
index 0000000000..871aee3672
--- /dev/null
+++ b/scripts/ci/pre_commit/common_precommit_utils.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pathlib import Path
+from typing import List
+
+
+def insert_documentation(file_path: Path, content: List[str], header: str,
footer: str):
+ text = file_path.read_text().splitlines(keepends=True)
+ replacing = False
+ result: List[str] = []
+ for line in text:
+ if line.startswith(header):
+ replacing = True
+ result.append(line)
+ result.extend(content)
+ if line.startswith(footer):
+ replacing = False
+ if not replacing:
+ result.append(line)
+ file_path.write_text("".join(result))
diff --git a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names.py
b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names.py
deleted file mode 100755
index dfd83daf86..0000000000
--- a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names.py
+++ /dev/null
@@ -1,166 +0,0 @@
-#!/usr/bin/env python3
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Module to check pre-commit hook names for length
-"""
-import argparse
-import sys
-
-import yaml
-
-try:
- from yaml import CSafeLoader as SafeLoader
-except ImportError:
- from yaml import SafeLoader # type: ignore
-
-from functools import lru_cache
-from pathlib import Path
-from typing import Any, Dict
-
-
-def main() -> int:
- parser = argparse.ArgumentParser()
- parser.add_argument('--max-length', help="Max length for hook names")
- args = parser.parse_args()
- max_length = int(args.max_length) or 70
-
- retval = 0
-
- with open('.pre-commit-config.yaml', 'rb') as f:
- content = yaml.load(f, SafeLoader)
- errors, ids = get_errors_and_ids(content, max_length)
- ids.append("all")
- ids.sort()
- prepare_pre_commit_ids_py_file(ids)
- if errors:
- retval = 1
- print(f"found pre-commit hook names with length exceeding {max_length}
characters")
- print("move add details in description if necessary")
- for hook_name in errors:
- print(f" * '{hook_name}': length {len(hook_name)}")
- return retval
-
-
-def get_errors_and_ids(content, max_length):
- errors = []
- ids = set()
- for repo in content['repos']:
- for hook in repo['hooks']:
- if 'id' in hook:
- id = hook['id']
- ids.add(id)
- if 'name' not in hook:
- continue
- name = hook['name']
- if len(name) > max_length:
- errors.append(name)
- return errors, list(ids)
-
-
-def render_template(
- searchpath: Path,
- template_name: str,
- context: Dict[str, Any],
- extension: str,
- autoescape: bool = True,
- keep_trailing_newline: bool = False,
-) -> str:
- """
- Renders template based on its name. Reads the template from
<name>_TEMPLATE.md.jinja2 in current dir.
- :param template_name: name of the template to use
- :param context: Jinja2 context
- :param extension: Target file extension
- :param autoescape: Whether to autoescape HTML
- :param keep_trailing_newline: Whether to keep the newline in rendered
output
- :return: rendered template
- """
- import jinja2
-
- template_loader = jinja2.FileSystemLoader(searchpath=searchpath)
- template_env = jinja2.Environment(
- loader=template_loader,
- undefined=jinja2.StrictUndefined,
- autoescape=autoescape,
- keep_trailing_newline=keep_trailing_newline,
- )
- template =
template_env.get_template(f"{template_name}_TEMPLATE{extension}.jinja2")
- content: str = template.render(context)
- return content
-
-
-# def get_precommit_jinja_context(pre_commit_ids: List) -> Dict[str, List]:
-# context: Dict[str, List] = {"PRE_COMMIT_IDS": pre_commit_ids}
-# return context
-
-
-def get_target_precommit_parent_dir():
- airflow_source = Path.cwd()
- pre_commit_ids_file = Path(airflow_source) / "dev" / "breeze" / "src" /
"airflow_breeze"
- return pre_commit_ids_file
-
-
-def get_breeze_pyproject_toml_dir():
- airflow_source = Path.cwd()
- breeze_pyproject_toml_file = Path(airflow_source) / "dev" / "breeze"
- return breeze_pyproject_toml_file
-
-
-@lru_cache(maxsize=None)
-def black_mode():
- from black import Mode, parse_pyproject_toml,
target_version_option_callback
-
- config = parse_pyproject_toml(Path(get_breeze_pyproject_toml_dir(),
"pyproject.toml"))
-
- target_versions = set(
- target_version_option_callback(None, None,
tuple(config.get('target_version', ()))),
- )
-
- return Mode(
- target_versions=target_versions,
- line_length=bool(config.get('line_length', Mode.line_length)),
- is_pyi=bool(config.get('is_pyi', Mode.is_pyi)),
- string_normalization=not bool(config.get('skip_string_normalization',
not Mode.string_normalization)),
- preview=bool(config.get('preview', Mode.preview)),
- )
-
-
-def black_format(content) -> str:
- from black import format_str
-
- return format_str(content, mode=black_mode())
-
-
-def prepare_pre_commit_ids_py_file(pre_commit_ids):
- pre_commit_ids_template_name = "pre_commit_ids"
- pre_commit_ids_file_path = Path(get_target_precommit_parent_dir(),
"pre_commit_ids.py")
-
- get_precommit_id_content = render_template(
- searchpath=get_target_precommit_parent_dir(),
- template_name=pre_commit_ids_template_name,
- context={"PRE_COMMIT_IDS": pre_commit_ids},
- extension='.py',
- autoescape=False,
- keep_trailing_newline=True,
- )
- with open(pre_commit_ids_file_path, "wt") as pre_commit_ids_file:
- pre_commit_ids_file.write(black_format(get_precommit_id_content))
-
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
new file mode 100755
index 0000000000..1eb368c86e
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Module to check pre-commit hook names for length
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.absolute())) # make sure
common_precommit_utils is imported
+
+from collections import defaultdict # noqa: E402
+from functools import lru_cache # noqa: E402
+from typing import Any, Dict, List, Tuple # noqa: E402
+
+import yaml # noqa: E402
+from common_precommit_utils import insert_documentation # noqa: E402
+from rich.console import Console # noqa: E402
+from tabulate import tabulate # noqa: E402
+
+console = Console(width=400, color_system="standard")
+
+AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].absolute()
+AIRFLOW_BREEZE_SOURCES_PATH = AIRFLOW_SOURCES_PATH / "dev" / "breeze"
+PRE_COMMIT_IDS_PATH = AIRFLOW_BREEZE_SOURCES_PATH / "src" / "airflow_breeze" /
"pre_commit_ids.py"
+PRE_COMMIT_YAML_FILE = AIRFLOW_SOURCES_PATH / ".pre-commit-config.yaml"
+
+
+def get_errors_and_hooks(content: Any, max_length: int) -> Tuple[List[str],
Dict[str, List[str]], List[str]]:
+ errors = []
+ hooks: Dict[str, List[str]] = defaultdict(list)
+ needs_image = False
+ image_hooks = []
+ for repo in content['repos']:
+ for hook in repo['hooks']:
+ if 'id' in hook:
+ hook_id = hook['id']
+ else:
+ errors.append(f"The id is missing in {hook}")
+ continue
+ if hook_id == 'mypy':
+ needs_image = True
+ if 'name' not in hook:
+ errors.append(f"Name is missing in hook {hook_id} in
{PRE_COMMIT_YAML_FILE}. Please add it!")
+ continue
+ name = hook['name']
+ if len(name) > max_length:
+ errors.append(
+ f"Name is too long for hook {hook_id} in
{PRE_COMMIT_YAML_FILE}. " f"Please shorten it!"
+ )
+ continue
+ hooks[hook_id].append(name)
+ if needs_image:
+ image_hooks.append(hook_id)
+ return errors, hooks, image_hooks
+
+
+def render_template(
+ searchpath: Path,
+ template_name: str,
+ context: Dict[str, Any],
+ extension: str,
+ autoescape: bool = True,
+ keep_trailing_newline: bool = False,
+) -> str:
+ """
+ Renders template based on its name. Reads the template from
<name>_TEMPLATE.md.jinja2 in current dir.
+ :param searchpath: Path to search images in
+ :param template_name: name of the template to use
+ :param context: Jinja2 context
+ :param extension: Target file extension
+ :param autoescape: Whether to autoescape HTML
+ :param keep_trailing_newline: Whether to keep the newline in rendered
output
+ :return: rendered template
+ """
+ import jinja2
+
+ template_loader = jinja2.FileSystemLoader(searchpath=searchpath)
+ template_env = jinja2.Environment(
+ loader=template_loader,
+ undefined=jinja2.StrictUndefined,
+ autoescape=autoescape,
+ keep_trailing_newline=keep_trailing_newline,
+ )
+ template =
template_env.get_template(f"{template_name}_TEMPLATE{extension}.jinja2")
+ content: str = template.render(context)
+ return content
+
+
+@lru_cache(maxsize=None)
+def black_mode():
+ from black import Mode, parse_pyproject_toml,
target_version_option_callback
+
+ config = parse_pyproject_toml(AIRFLOW_BREEZE_SOURCES_PATH /
"pyproject.toml")
+
+ target_versions = set(
+ target_version_option_callback(None, None,
tuple(config.get('target_version', ()))),
+ )
+
+ return Mode(
+ target_versions=target_versions,
+ line_length=bool(config.get('line_length', Mode.line_length)),
+ is_pyi=bool(config.get('is_pyi', Mode.is_pyi)),
+ string_normalization=not bool(config.get('skip_string_normalization',
not Mode.string_normalization)),
+ preview=bool(config.get('preview', Mode.preview)),
+ )
+
+
+def black_format(content) -> str:
+ from black import format_str
+
+ return format_str(content, mode=black_mode())
+
+
+def prepare_pre_commit_ids_py_file(pre_commit_ids):
+ PRE_COMMIT_IDS_PATH.write_text(
+ black_format(
+ render_template(
+ searchpath=AIRFLOW_BREEZE_SOURCES_PATH / "src" /
"airflow_breeze",
+ template_name="pre_commit_ids",
+ context={"PRE_COMMIT_IDS": pre_commit_ids},
+ extension='.py',
+ autoescape=False,
+ keep_trailing_newline=True,
+ )
+ )
+ )
+
+
+def update_static_checks_array(hooks: Dict[str, List[str]], image_hooks:
List[str]):
+ rows = []
+ hook_ids = list(hooks.keys())
+ hook_ids.sort()
+ for hook_id in hook_ids:
+ hook_description = hooks[hook_id]
+ formatted_hook_description = (
+ hook_description[0] if len(hook_description) == 1 else "* " + "\n*
".join(hook_description)
+ )
+ rows.append((hook_id, formatted_hook_description, " * " if hook_id in
image_hooks else " "))
+ formatted_table = "\n" + tabulate(rows, tablefmt="grid", headers=("ID",
"Description", 'Image')) + "\n\n"
+ insert_documentation(
+ file_path=AIRFLOW_SOURCES_PATH / "STATIC_CODE_CHECKS.rst",
+ content=formatted_table.splitlines(keepends=True),
+ header=" .. BEGIN AUTO-GENERATED STATIC CHECK LIST",
+ footer=" .. END AUTO-GENERATED STATIC CHECK LIST",
+ )
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--max-length', help="Max length for hook names")
+ args = parser.parse_args()
+ max_length = int(args.max_length) or 70
+ content = yaml.safe_load(PRE_COMMIT_YAML_FILE.read_text())
+ errors, hooks, image_hooks = get_errors_and_hooks(content, max_length)
+ if errors:
+ for error in errors:
+ console.print(f"* [red]{error}[/]")
+ sys.exit(1)
+ ids = list(hooks.keys())
+ ids.append("all")
+ ids.sort()
+ prepare_pre_commit_ids_py_file(ids)
+ update_static_checks_array(hooks, image_hooks)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
b/scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
deleted file mode 100755
index 54518ff273..0000000000
--- a/scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-set -euo pipefail
-
-export PRINT_INFO_FROM_SCRIPTS="false"
-
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
-cd "${AIRFLOW_SOURCES}"
-
-. ./breeze-complete
-
-PRE_COMMIT_CONFIG_FILE="${AIRFLOW_SOURCES}/.pre-commit-config.yaml"
-readonly PRE_COMMIT_CONFIG_FILE
-
-STATIC_CODE_CHECKS_FILE="${AIRFLOW_SOURCES}/STATIC_CODE_CHECKS.rst"
-
-all_pre_commits=$(grep "id:" <"${PRE_COMMIT_CONFIG_FILE}" | sort |uniq | awk
'{ print $3 }')
-
-error="false"
-for pre_commit in ${all_pre_commits}
-do
- if ! grep -q "${pre_commit}" "${STATIC_CODE_CHECKS_FILE}"; then
- error="true"
- echo
- echo """
-${COLOR_RED}ERROR: Pre-commit ${pre_commit} is not described in
${STATIC_CODE_CHECKS_FILE}
-ERROR: Pre-commit ${pre_commit} is not described in ${STATIC_CODE_CHECKS_FILE}
-
-FIX: Please add ${pre_commit} in the table in the 'Pre-commit hooks' chapter
in ${STATIC_CODE_CHECKS_FILE}
-${COLOR_RESET}
-"""
- echo
- fi
- # shellcheck disable=SC2154
- if [[ ! ${_breeze_allowed_static_checks} == *${pre_commit}* ]]; then
- error="true"
- echo """
-${COLOR_RED}ERROR:: Pre-commit ${pre_commit} is missing in
_breeze_allowed_static_checks variable in breeze-complete
-
-FIX: Please add ${pre_commit} in the table in the
'_breeze_allowed_static_checks' constant in ${AIRFLOW_SOURCES}/breeze-complete
-${COLOR_RESET}
-"""
- fi
-done
-
-if [[ ${error} == "true" ]]; then
- echo
- echo "${COLOR_RED}ERROR: Some pre-commits are not synchronized! Please
fix the errors above! ${COLOR_RESET}"
- echo
- exit 1
-else
- echo
- echo "${COLOR_GREEN}OK. All pre-commits are synchronized! ${COLOR_RESET}"
- echo
-fi
diff --git a/scripts/ci/pre_commit/pre_commit_insert_extras.py
b/scripts/ci/pre_commit/pre_commit_insert_extras.py
index 1f4c1b73fd..79c0a6e989 100755
--- a/scripts/ci/pre_commit/pre_commit_insert_extras.py
+++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py
@@ -18,13 +18,14 @@
import sys
from pathlib import Path
from textwrap import wrap
-from typing import List
AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].absolute()
-sys.path.insert(0, str(AIRFLOW_SOURCES_DIR))
+sys.path.insert(0, str(Path(__file__).parent.absolute())) # make sure
common_precommit_utils is imported
+sys.path.insert(0, str(AIRFLOW_SOURCES_DIR)) # make sure setup is imported
from Airflow
# flake8: noqa: F401
+from common_precommit_utils import insert_documentation # isort: skip
from setup import EXTRAS_REQUIREMENTS # isort:skip
sys.path.append(str(AIRFLOW_SOURCES_DIR))
@@ -45,22 +46,6 @@ DEFAULT_EXTRAS = (
)
-def insert_documentation(file_path: Path, content: List[str], header: str,
footer: str):
- text = file_path.read_text().splitlines(keepends=True)
- replacing = False
- result: List[str] = []
- for line in text:
- if line.startswith(header):
- replacing = True
- result.append(line)
- result.extend(content)
- if line.startswith(footer):
- replacing = False
- if not replacing:
- result.append(line)
- file_path.write_text("".join(result))
-
-
if __name__ == '__main__':
install_file_path = AIRFLOW_SOURCES_DIR / 'INSTALL'
contributing_file_path = AIRFLOW_SOURCES_DIR / 'CONTRIBUTING.rst'
diff --git a/scripts/ci/static_checks/run_basic_static_checks.sh
b/scripts/ci/static_checks/run_basic_static_checks.sh
deleted file mode 100755
index fb5256e78d..0000000000
--- a/scripts/ci/static_checks/run_basic_static_checks.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$(dirname "${BASH_SOURCE[0]}")/../libraries/_script_init.sh"
-
-if [[ -f "${BUILD_CACHE_DIR}/.skip_tests" ]]; then
- echo
- echo "Skip tests"
- echo
- exit
-fi
-
-if (($# < 1)); then
- echo
- echo "Missing SHAs of incoming commit!"
- echo
- exit 1
-fi
-
-COMMIT_SHA="${1}"
-shift
-
-python -m pip install --user pre-commit \
- --constraint
"https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt";
-
-export PATH=~/.local/bin:${PATH}
-
-if [[ $# == "0" ]]; then
- pre-commit run --all-files --show-diff-on-failure --color always \
- --from-ref "${COMMIT_SHA}^" --to-ref "${COMMIT_SHA}"
-else
- for pre_commit_check in "${@}"; do
- pre-commit run "${pre_commit_check}" --all-files
--show-diff-on-failure --color always \
- --from-ref "${COMMIT_SHA}^" --to-ref "${COMMIT_SHA}"
- done
-fi
diff --git a/scripts/ci/static_checks/run_static_checks.sh
b/scripts/ci/static_checks/run_static_checks.sh
deleted file mode 100755
index 17bb0fb4d0..0000000000
--- a/scripts/ci/static_checks/run_static_checks.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
-if [[ -f "${BUILD_CACHE_DIR}/.skip_tests" ]]; then
- echo
- echo "Skip tests"
- echo
- exit
-fi
-
-python -m pip install --user pre-commit \
- --constraint
"https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt";
-
-export PATH=~/.local/bin:${PATH}
-
-if [[ ${DO_NOT_FAIL_ON_ERROR=} != "" ]]; then
- echo
- echo "Skip failing on error"
- echo
- set +e
-fi
-
-if [[ $# == "0" ]]; then
- pre-commit run --all-files --show-diff-on-failure --color always
-else
- for pre_commit_check in "${@}"
- do
- pre-commit run "${pre_commit_check}" --all-files
--show-diff-on-failure --color always
- done
-fi
-
-if [[ ${DO_NOT_FAIL_ON_ERROR=} != "" ]]; then
- exit 0
-fi
