Repository: incubator-airflow Updated Branches: refs/heads/master f1df3de9b -> 976fd1245
[AIRFLOW-2123] Install CI dependencies from setup.py Install the dependencies from setup.py so we keep all the dependencies in one single place Closes #3054 from Fokko/fd-fix-ci-2 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/976fd124 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/976fd124 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/976fd124 Branch: refs/heads/master Commit: 976fd1245a981b37957e4e35367b0e504d8e3d67 Parents: f1df3de Author: Fokko Driesprong <fokkodriespr...@godatadriven.com> Authored: Mon Mar 5 22:46:07 2018 +0000 Committer: Ash Berlin-Taylor <ash.berlin-tay...@annalect.com> Committed: Mon Mar 5 22:46:45 2018 +0000 ---------------------------------------------------------------------- scripts/ci/requirements.txt | 97 ---------------------------------------- scripts/ci/travis_script.sh | 2 + setup.py | 17 +++++-- tox.ini | 4 +- 4 files changed, 17 insertions(+), 103 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/976fd124/scripts/ci/requirements.txt ---------------------------------------------------------------------- diff --git a/scripts/ci/requirements.txt b/scripts/ci/requirements.txt deleted file mode 100644 index 9c028d5..0000000 --- a/scripts/ci/requirements.txt +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -alembic -azure-storage>=0.34.0 -bcrypt -bleach -boto -boto3 -celery -cgroupspy -chartkick -cloudant<2.0 -coverage -coveralls -croniter>=0.3.17 -cryptography -datadog -dill -distributed -docker-py -filechunkio -flake8 -flask -flask-admin -flask-bcrypt -flask-cache -flask-login==0.2.11 -Flask-WTF -flower -freezegun -future -google-api-python-client>=1.5.0,<1.6.0 -gunicorn -hdfs -hive-thrift-py -impyla -ipython -jaydebeapi -jinja2<2.9.0 -jira -ldap3 -lxml -markdown -mock -moto==1.1.19 -mysqlclient -nose -nose-exclude -nose-ignore-docstring==0.2 -nose-timer -oauth2client>=2.0.2,<2.1.0 -pandas -pandas-gbq -parameterized -paramiko>=2.1.1 -pendulum>=1.3.2 -psutil>=4.2.0, <5.0.0 -psycopg2 -pygments -pyhive -pykerberos -PyOpenSSL -PySmbClient -python-daemon -python-dateutil -python-jenkins -qds-sdk>=1.9.6 -redis -rednose -requests -requests-kerberos -requests_mock -sendgrid -setproctitle -slackclient -sphinx -sphinx-argparse -Sphinx-PyPI-upload -sphinx_rtd_theme -sqlalchemy>=1.1.15, <1.2.0 -statsd -thrift -thrift_sasl -unicodecsv -zdesk -kubernetes http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/976fd124/scripts/ci/travis_script.sh ---------------------------------------------------------------------- diff --git a/scripts/ci/travis_script.sh b/scripts/ci/travis_script.sh index 86c086a..8766e94 100755 --- a/scripts/ci/travis_script.sh +++ b/scripts/ci/travis_script.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one * # or more contributor license agreements. See the NOTICE file * # distributed with this work for additional information * http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/976fd124/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index d3f48e3..254aa3e 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ logger = logging.getLogger(__name__) version = imp.load_source( 'airflow.version', os.path.join('airflow', 'version.py')).version +PY3 = sys.version_info[0] == 3 class Tox(TestCommand): user_options = [('tox-args=', None, "Arguments to pass to tox")] @@ -153,8 +154,7 @@ ldap = ['ldap3>=0.9.9.1'] kerberos = ['pykerberos>=1.1.13', 'requests_kerberos>=0.10.0', 'thrift_sasl>=0.2.0', - 'snakebite[kerberos]>=2.7.8', - 'kerberos>=1.2.5'] + 'snakebite[kerberos]>=2.7.8'] password = [ 'bcrypt>=2.0.0', 'flask-bcrypt>=0.7.1', @@ -166,6 +166,8 @@ redis = ['redis>=2.10.5'] kubernetes = ['kubernetes>=3.0.0', 'cryptography>=2.0.0'] +zendesk = ['zdesk'] + all_dbs = postgres + mysql + hive + mssql + hdfs + vertica + cloudant devel = [ 'click', @@ -185,9 +187,15 @@ devel = [ ] devel_minreq = devel + kubernetes + mysql + doc + password + s3 + cgroups devel_hadoop = devel_minreq + hive + hdfs + webhdfs + kerberos -devel_all = (devel + all_dbs + doc + samba + s3 + slack + crypto + oracle + docker + ssh + - kubernetes) +devel_all = (sendgrid + devel + all_dbs + doc + samba + s3 + slack + crypto + oracle + + docker + ssh + kubernetes + celery + azure + redis + gcp_api + datadog + + zendesk + jdbc + ldap + kerberos + password + webhdfs + jenkins) +# Snakebite is not Python 3 compatible :'( +if PY3: + devel_ci = list(filter(lambda package: 'snakebite' not in package, devel_all)) +else: + devel_ci = devel_all def do_setup(): write_version() @@ -243,6 +251,7 @@ def do_setup(): ], extras_require={ 'all': devel_all, + 'devel_ci': devel_ci, 'all_dbs': all_dbs, 'async': async, 'azure': azure, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/976fd124/tox.ini ---------------------------------------------------------------------- diff --git a/tox.ini b/tox.ini index e875694..e44d7ac 100644 --- a/tox.ini +++ b/tox.ini @@ -61,8 +61,8 @@ passenv = KRB5_KTNAME commands = - pip wheel -w {homedir}/.wheelhouse -f {homedir}/.wheelhouse -r scripts/ci/requirements.txt - pip install --find-links={homedir}/.wheelhouse --no-index -r scripts/ci/requirements.txt + pip wheel -w {homedir}/.wheelhouse -f {homedir}/.wheelhouse -e .[devel_ci] + pip install --find-links={homedir}/.wheelhouse --no-index -e .[devel_ci] sudo {toxinidir}/scripts/ci/setup_kdc.sh {toxinidir}/scripts/ci/setup_env.sh {toxinidir}/scripts/ci/ldap.sh
