This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 7883885 Move setup order check back to pre-commit (#9010)
7883885 is described below
commit 7883885e6c579ea8f8e0a09fdfee524ca0a8cfd0
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue May 26 12:58:24 2020 +0200
Move setup order check back to pre-commit (#9010)
* Move setup order check back to pre-commit
The order check used to be working from pre-commit but then it
was moved to be regular test case. That was a mistake
The test is super-fast and actually making it use assertEquals
was not very useful and it was very late when you found it out.
I changed it to be normal python script which made it works again
(it did not work when it was a test because pre-commit does not
run tests - it runs python scripts).
The messages printed now are much more informative as well.
---
.pre-commit-config.yaml | 4 +-
requirements/requirements-python3.6.txt | 4 +-
requirements/requirements-python3.7.txt | 8 +-
requirements/setup-3.6.md5 | 2 +-
requirements/setup-3.7.md5 | 2 +-
scripts/ci/pre_commit_check_order_setup.py | 165 +++++++++++++++++++++++++++++
setup.py | 6 +-
tests/test_order_setup.py | 131 -----------------------
8 files changed, 178 insertions(+), 144 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7b1081f..996d7a4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -195,7 +195,7 @@ metastore_browser/templates/.*\\.html$|.*\\.jinja2"
files: ^setup.py$
pass_filenames: false
require_serial: true
- entry: tests/test_order_setup.py
+ entry: ./scripts/ci/pre_commit_check_order_setup.py
- id: update-breeze-file
name: Update output of breeze command in BREEZE.rst
entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh"
@@ -315,7 +315,7 @@ metastore_browser/templates/.*\\.html$|.*\\.jinja2"
files: "config.yml$|default_airflow.cfg$|default.cfg$"
pass_filenames: false
require_serial: false
- entry: scripts/ci/pre_commit_yaml_to_cfg.py
+ entry: ./scripts/ci/pre_commit_yaml_to_cfg.py
additional_dependencies: ['pyyaml']
- id: mypy
name: Run mypy
diff --git a/requirements/requirements-python3.6.txt
b/requirements/requirements-python3.6.txt
index 4751abf6..abf6b03 100644
--- a/requirements/requirements-python3.6.txt
+++ b/requirements/requirements-python3.6.txt
@@ -9,7 +9,7 @@ Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OAuthlib==0.9.5
Flask-OpenID==1.2.5
-Flask-SQLAlchemy==2.4.1
+Flask-SQLAlchemy==2.4.2
Flask-WTF==0.14.3
Flask==1.1.2
GitPython==3.1.2
@@ -22,7 +22,7 @@ Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.2
PyJWT==1.7.1
-PyNaCl==1.3.0
+PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.6.1
diff --git a/requirements/requirements-python3.7.txt
b/requirements/requirements-python3.7.txt
index 158f847..f010e83 100644
--- a/requirements/requirements-python3.7.txt
+++ b/requirements/requirements-python3.7.txt
@@ -9,7 +9,7 @@ Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OAuthlib==0.9.5
Flask-OpenID==1.2.5
-Flask-SQLAlchemy==2.4.1
+Flask-SQLAlchemy==2.4.2
Flask-WTF==0.14.3
Flask==1.1.2
GitPython==3.1.2
@@ -22,7 +22,7 @@ Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.2
PyJWT==1.7.1
-PyNaCl==1.3.0
+PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.6.1
@@ -45,7 +45,7 @@ apispec==1.3.3
appdirs==1.4.4
argcomplete==1.11.1
asn1crypto==1.3.0
-astroid==2.3.3
+astroid==2.4.1
async-generator==1.10
async-timeout==3.0.1
atlasclient==1.0.0
@@ -371,7 +371,7 @@ virtualenv==20.0.21
watchtower==0.7.3
wcwidth==0.1.9
websocket-client==0.57.0
-wrapt==1.11.2
+wrapt==1.12.1
xmltodict==0.12.0
yamllint==1.23.0
yandexcloud==0.39.0
diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5
index 10fe97a..aa56523 100644
--- a/requirements/setup-3.6.md5
+++ b/requirements/setup-3.6.md5
@@ -1 +1 @@
-aa30d51f3388ba898dfeeabc4df5f16b /opt/airflow/setup.py
+92aca7dc7f21d787c2e1a638d0355a00 /opt/airflow/setup.py
diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5
index 10fe97a..aa56523 100644
--- a/requirements/setup-3.7.md5
+++ b/requirements/setup-3.7.md5
@@ -1 +1 @@
-aa30d51f3388ba898dfeeabc4df5f16b /opt/airflow/setup.py
+92aca7dc7f21d787c2e1a638d0355a00 /opt/airflow/setup.py
diff --git a/scripts/ci/pre_commit_check_order_setup.py
b/scripts/ci/pre_commit_check_order_setup.py
new file mode 100755
index 0000000..5eb7cf9
--- /dev/null
+++ b/scripts/ci/pre_commit_check_order_setup.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Test for an order of dependencies in setup.py
+"""
+
+import os
+import re
+import sys
+from os.path import abspath, dirname
+from typing import List
+
+errors = []
+
+
+def _check_list_sorted(the_list: List[str], message: str) -> None:
+ sorted_list = sorted(the_list)
+ if the_list == sorted_list:
+ print(f"{message} is ok")
+ return
+ i = 0
+ while sorted_list[i] == the_list[i]:
+ i += 1
+ print(f"{message} NOK")
+ errors.append(f"ERROR in {message}. First wrongly sorted element"
+ f" {the_list[i]}. Should be {sorted_list[i]}")
+
+
+def setup() -> str:
+ setup_py_file_path = abspath(os.path.join(dirname(__file__), os.pardir,
os.pardir, 'setup.py'))
+ with open(setup_py_file_path) as setup_file:
+ setup_context = setup_file.read()
+ return setup_context
+
+
+def check_main_dependent_group(setup_context: str) -> None:
+ """
+ Test for an order of dependencies groups between mark
+ '# Start dependencies group' and '# End dependencies group' in setup.py
+ """
+ pattern_main_dependent_group = re.compile(
+ '# Start dependencies group\n(.*)# End dependencies group', re.DOTALL)
+ main_dependent_group =
pattern_main_dependent_group.findall(setup_context)[0]
+
+ pattern_sub_dependent = re.compile(' = \\[.*?\\]\n', re.DOTALL)
+ main_dependent = pattern_sub_dependent.sub(',', main_dependent_group)
+
+ src = main_dependent.strip(',').split(',')
+ _check_list_sorted(src, "Order of dependencies")
+
+
+def check_sub_dependent_group(setup_context: str) -> None:
+ """
+ Test for an order of each dependencies groups declare like
+ `^dependent_group_name = [.*?]\n` in setup.py
+ """
+ pattern_dependent_group_name = re.compile('^(\\w+) = \\[', re.MULTILINE)
+ dependent_group_names = pattern_dependent_group_name.findall(setup_context)
+
+ pattern_dependent_version = re.compile('[~|><=;].*')
+
+ for group_name in dependent_group_names:
+ pattern_sub_dependent = re.compile(
+ '{group_name} = \\[(.*?)\\]'.format(group_name=group_name),
re.DOTALL)
+ sub_dependent = pattern_sub_dependent.findall(setup_context)[0]
+ pattern_dependent = re.compile('\'(.*?)\'')
+ dependent = pattern_dependent.findall(sub_dependent)
+
+ src = [pattern_dependent_version.sub('', p) for p in dependent]
+ _check_list_sorted(src, f"Order of sub-dependencies group:
{group_name}")
+
+
+def check_alias_dependent_group(setup_context: str) -> None:
+ """
+ Test for an order of each dependencies groups declare like
+ `alias_dependent_group = dependent_group_1 + ... + dependent_group_n` in
setup.py
+ """
+ pattern = re.compile('^\\w+ = (\\w+ \\+.*)', re.MULTILINE)
+ dependents = pattern.findall(setup_context)
+
+ for dependent in dependents:
+ src = dependent.split(' + ')
+ _check_list_sorted(src, f"Order of alias dependencies group:
{dependent}")
+
+
+def check_install_and_setup_requires(setup_context: str) -> None:
+ """
+ Test for an order of dependencies in function do_setup section
+ install_requires and setup_requires in setup.py
+ """
+ pattern_install_and_setup_requires = re.compile(
+ '(setup_requires) ?= ?\\[(.*?)\\]', re.DOTALL)
+ install_and_setup_requires =
pattern_install_and_setup_requires.findall(setup_context)
+
+ for dependent_requires in install_and_setup_requires:
+ pattern_dependent = re.compile('\'(.*?)\'')
+ dependent = pattern_dependent.findall(dependent_requires[1])
+ pattern_dependent_version = re.compile('[~|><=;].*')
+
+ src = [pattern_dependent_version.sub('', p) for p in dependent]
+ _check_list_sorted(src, f"Order of dependencies in do_setup section:
{dependent_requires[0]}")
+
+
+def check_extras_require(setup_context: str) -> None:
+ """
+ Test for an order of dependencies in function do_setup section
+ extras_require in setup.py
+ """
+ pattern_extras_requires = re.compile(
+ r'EXTRAS_REQUIREMENTS: Dict\[str, Iterable\[str\]] = {(.*?)}',
re.DOTALL)
+ extras_requires = pattern_extras_requires.findall(setup_context)[0]
+
+ pattern_dependent = re.compile('\'(.*?)\'')
+ src = pattern_dependent.findall(extras_requires)
+ _check_list_sorted(src, "Order of dependencies in: extras_require")
+
+
+def check_provider_requirements(setup_context: str) -> None:
+ """
+ Test for an order of dependencies in function do_setup section
+ providers_require in setup.py
+ """
+ pattern_extras_requires = re.compile(
+ r'PROVIDERS_REQUIREMENTS: Dict\[str, Iterable\[str\]\] = {(.*?)}',
re.DOTALL)
+ extras_requires = pattern_extras_requires.findall(setup_context)[0]
+
+ pattern_dependent = re.compile('"(.*?)"')
+ src = pattern_dependent.findall(extras_requires)
+ _check_list_sorted(src, "Order of dependencies in: providers_require")
+
+
+if __name__ == '__main__':
+ setup_context_main = setup()
+ check_main_dependent_group(setup_context_main)
+ check_alias_dependent_group(setup_context_main)
+ check_sub_dependent_group(setup_context_main)
+ check_install_and_setup_requires(setup_context_main)
+ check_extras_require(setup_context_main)
+ check_provider_requirements(setup_context_main)
+
+ print()
+ print()
+ for error in errors:
+ print(error)
+
+ print()
+
+ if errors:
+ sys.exit(1)
diff --git a/setup.py b/setup.py
index 30c9423..7c5d4cd 100644
--- a/setup.py
+++ b/setup.py
@@ -170,8 +170,8 @@ def write_version(filename: str = os.path.join(*[my_dir,
"airflow", "git_version
file.write(text)
-# 'Start dependencies group' and 'Start dependencies group' are mark for
./test/test_order_setup.py
-# If you change this mark you should also change ./test/test_order_setup.py
function test_main_dependent_group
+# 'Start dependencies group' and 'Start dependencies group' are mark for
./scripts/ci/check_order_setup.py
+# If you change this mark you should also change
./scripts/ci/check_order_setup.py
# Start dependencies group
amazon = [
'boto3>=1.12.0,<2.0.0',
@@ -502,8 +502,8 @@ PROVIDERS_REQUIREMENTS: Dict[str, Iterable[str]] = {
"dingding": [],
"discord": [],
"docker": docker,
- "email": [],
"elasticsearch": [],
+ "email": [],
"exasol": exasol,
"facebook": facebook,
"ftp": [],
diff --git a/tests/test_order_setup.py b/tests/test_order_setup.py
deleted file mode 100755
index a7cbea5..0000000
--- a/tests/test_order_setup.py
+++ /dev/null
@@ -1,131 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Test for an order of dependencies in setup.py
-"""
-
-import os
-import re
-import unittest
-
-
-class TestOrderSetup(unittest.TestCase):
-
- def setUp(self):
- current_dir = os.path.dirname(os.path.abspath(__file__))
- parent_dir = os.path.dirname(current_dir)
- self.setup_file =
open('{parent_dir}/setup.py'.format(parent_dir=parent_dir))
- self.setup_context = self.setup_file.read()
-
- def tearDown(self):
- self.setup_file.close()
-
- def test_main_dependent_group(self):
- """
- Test for an order of dependencies groups between mark
- '# Start dependencies group' and '# End dependencies group' in setup.py
- """
- pattern_main_dependent_group = re.compile(
- '# Start dependencies group\n(.*)# End dependencies group',
re.DOTALL)
- main_dependent_group =
pattern_main_dependent_group.findall(self.setup_context)[0]
-
- pattern_sub_dependent = re.compile(' = \\[.*?\\]\n', re.DOTALL)
- main_dependent = pattern_sub_dependent.sub(',', main_dependent_group)
-
- src = main_dependent.strip(',').split(',')
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_sub_dependent_group(self):
- """
- Test for an order of each dependencies groups declare like
- `^dependent_group_name = [.*?]\n` in setup.py
- """
- pattern_dependent_group_name = re.compile('^(\\w+) = \\[',
re.MULTILINE)
- dependent_group_names =
pattern_dependent_group_name.findall(self.setup_context)
-
- pattern_dependent_version = re.compile('[~|>|<|=|;].*')
- for group_name in dependent_group_names:
- pattern_sub_dependent = re.compile(
- '{group_name} = \\[(.*?)\\]'.format(group_name=group_name),
re.DOTALL)
- sub_dependent =
pattern_sub_dependent.findall(self.setup_context)[0]
- pattern_dependent = re.compile('\'(.*?)\'')
- dependent = pattern_dependent.findall(sub_dependent)
-
- src = [pattern_dependent_version.sub('', p) for p in dependent]
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_alias_dependent_group(self):
- """
- Test for an order of each dependencies groups declare like
- `alias_dependent_group = dependent_group_1 + ... + dependent_group_n`
in setup.py
- """
- pattern = re.compile('^\\w+ = (\\w+ \\+.*)', re.MULTILINE)
- dependents = pattern.findall(self.setup_context)
- for dependent in dependents:
- src = dependent.split(' + ')
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_install_and_setup_requires(self):
- """
- Test for an order of dependencies in function do_setup section
- install_requires and setup_requires in setup.py
- """
- pattern_install_and_setup_requires = re.compile(
- '(INSTALL_REQUIREMENTS|setup_requires) ?= ?\\[(.*?)\\]', re.DOTALL)
- install_and_setup_requires =
pattern_install_and_setup_requires.findall(self.setup_context)
-
- for dependent_requires in install_and_setup_requires:
- pattern_dependent = re.compile('\'(.*?)\'')
- dependent = pattern_dependent.findall(dependent_requires[1])
- pattern_dependent_version = re.compile('[~|>|<|=|;].*')
-
- src = [pattern_dependent_version.sub('', p) for p in dependent]
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_extras_require(self):
- """
- Test for an order of dependencies in function do_setup section
- extras_require in setup.py
- """
- pattern_extras_requires = re.compile(
- r'EXTRAS_REQUIREMENTS: Dict\[str, Iterable\[str\]] = {(.*?)}',
re.DOTALL)
- extras_requires =
pattern_extras_requires.findall(self.setup_context)[0]
-
- pattern_dependent = re.compile('\'(.*?)\'')
- src = pattern_dependent.findall(extras_requires)
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_provider_requirements(self):
- """
- Test for an order of dependencies in function do_setup section
- extras_require in setup.py
- """
- pattern_extras_requires = re.compile(
- r'PROVIDERS_REQUIREMENTS: Dict\[str, Iterable\[str\]\] = {(.*?)}',
re.DOTALL)
- extras_requires =
pattern_extras_requires.findall(self.setup_context)[0]
-
- pattern_dependent = re.compile('\'(.*?)\'')
- src = pattern_dependent.findall(extras_requires)
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
