[AIRFLOW-1314] Cleanup the config Closes #2414 from bloomberg:airflow-kubernetes-executor
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/a15b7c5b Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/a15b7c5b Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/a15b7c5b Branch: refs/heads/master Commit: a15b7c5b79ab90cc87f4da11c687aa883482716f Parents: d807830 Author: Daniel Imberman <[email protected]> Authored: Thu Apr 19 11:45:27 2018 -0700 Committer: Fokko Driesprong <[email protected]> Committed: Sun Apr 22 10:24:18 2018 +0200 ---------------------------------------------------------------------- .travis.yml | 8 +- airflow/bin/cli.py | 8 +- scripts/ci/kubernetes/docker/requirements.txt | 138 ++++++++++++++----- .../ci/kubernetes/kube/airflow.yaml.template | 32 ++++- .../ci/kubernetes/minikube/start_minikube.sh | 128 ++++++++++------- 5 files changed, 216 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a15b7c5b/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index daae12c..d59e885 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ env: - TOX_ENV=py35-backend_postgres - TOX_ENV=flake8 - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.8.0 - - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0 + - TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.9.0 matrix: exclude: - python: "3.5" @@ -78,11 +78,11 @@ matrix: env: TOX_ENV=flake8 - python: "3.5" env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.8.0 - - python: "3.5" - env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0 + - python: "2.7" + env: TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.9.0 allow_failures: - env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.8.0 - - env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0 + - env: TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.9.0 cache: directories: - $HOME/.wheelhouse/ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a15b7c5b/airflow/bin/cli.py ---------------------------------------------------------------------- diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py index 20e7496..a5936a9 100644 --- a/airflow/bin/cli.py +++ b/airflow/bin/cli.py @@ -21,8 +21,6 @@ from __future__ import print_function import logging -import reprlib - import os import socket import subprocess @@ -266,12 +264,8 @@ def pool(args): else: log.info(_tabulate(pools=pools)) -<<<<<<< HEAD:airflow/bin/cli/cli.py -@cli_utils.action_logging -def variables(args): -======= ->>>>>>> [AIRFLOW-1314] Rebasing against master:airflow/bin/cli.py +@cli_utils.action_logging def variables(args): if args.get: try: http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a15b7c5b/scripts/ci/kubernetes/docker/requirements.txt ---------------------------------------------------------------------- diff --git a/scripts/ci/kubernetes/docker/requirements.txt b/scripts/ci/kubernetes/docker/requirements.txt index 6b823d9..aa136e0 100644 --- a/scripts/ci/kubernetes/docker/requirements.txt +++ b/scripts/ci/kubernetes/docker/requirements.txt @@ -1,35 +1,103 @@ -alembic -bleach -configparser -croniter -dill -flask -flask-admin -flask-caching -flask-login -flask-swagger -flask-wtf -funcsigs -future -gitpython -gunicorn -iso8601 -jinja2 -lxml -markdown -pandas -pendulum -psutil -pygments -python-daemon -python-dateutil -python-nvd3 -requests -setproctitle -sqlalchemy -sqlalchemy-utc -tabulate -thrift -tzlocal -werkzeug -zope.deprecation +alembic==0.8.10 +asn1crypto==0.24.0 +aws-xray-sdk==0.95 +Babel==2.5.3 +bleach==2.1.2 +boto==2.48.0 +boto3==1.7.4 +botocore==1.10.4 +cachetools==2.0.1 +certifi==2018.4.16 +cffi==1.11.5 +chardet==3.0.4 +click==6.7 +colorama==0.3.9 +configparser==3.5.0 +cookies==2.2.1 +croniter==0.3.20 +cryptography==2.2.2 +defusedxml==0.5.0 +dill==0.2.7.1 +docker==3.2.1 +docker-pycreds==0.2.2 +docutils==0.14 +Flask==0.12.2 +Flask-Admin==1.4.1 +Flask-AppBuilder==1.10.0 +Flask-Babel==0.11.1 +Flask-Caching==1.3.3 +Flask-Login==0.2.11 +Flask-OpenID==1.2.5 +Flask-SQLAlchemy==2.1 +flask-swagger==0.2.13 +Flask-WTF==0.14.2 +freezegun==0.3.10 +funcsigs==1.0.0 +future==0.16.0 +gitdb2==2.0.3 +GitPython==2.1.9 +google-auth==1.4.1 +gunicorn==19.7.1 +html5lib==1.0.1 +idna==2.6 +ipaddress==1.0.22 +iso8601==0.1.12 +itsdangerous==0.24 +Jinja2==2.8.1 +jmespath==0.9.3 +jsondiff==1.1.1 +jsonpickle==0.9.6 +kubernetes==6.0.0 +lockfile==0.12.2 +lxml==3.8.0 +Mako==1.0.7 +Markdown==2.6.11 +MarkupSafe==1.0 +mock==2.0.0 +moto==1.3.3 +nose==1.3.7 +numpy==1.14.2 +oauthlib==2.0.7 +ordereddict==1.1 +pandas==0.22.0 +parameterized==0.6.1 +pbr==4.0.2 +pendulum==1.4.4 +psutil==4.4.2 +pyaml==17.12.1 +pyasn1==0.4.2 +pyasn1-modules==0.2.1 +pycparser==2.18 +Pygments==2.2.0 +python-daemon==2.1.2 +python-dateutil==2.6.1 +python-editor==1.0.3 +python-nvd3==0.15.0 +python-slugify==1.2.5 +python3-openid==3.1.0 +pytz==2018.4 +pytzdata==2018.4 +PyYAML==3.12 +requests==2.18.4 +requests-oauthlib==0.8.0 +responses==0.9.0 +rsa==3.4.2 +s3transfer==0.1.13 +setproctitle==1.1.10 +six==1.11.0 +smmap2==2.0.3 +SQLAlchemy==1.1.18 +SQLAlchemy-Utc==0.10.0 +tabulate==0.7.7 +thrift==0.11.0 +tzlocal==1.5.1 +unicodecsv==0.14.1 +Unidecode==1.0.22 +urllib3==1.22 +webencodings==0.5.1 +websocket-client==0.47.0 +Werkzeug==0.14.1 +wrapt==1.10.11 +WTForms==2.1 +xmltodict==0.11.0 +zope.deprecation==4.3.0 http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a15b7c5b/scripts/ci/kubernetes/kube/airflow.yaml.template ---------------------------------------------------------------------- diff --git a/scripts/ci/kubernetes/kube/airflow.yaml.template b/scripts/ci/kubernetes/kube/airflow.yaml.template index 3f791a1..6a57c24 100644 --- a/scripts/ci/kubernetes/kube/airflow.yaml.template +++ b/scripts/ci/kubernetes/kube/airflow.yaml.template @@ -194,8 +194,38 @@ data: sql_alchemy_conn = $SQL_ALCHEMY_CONN [scheduler] - dag_dir_list_interval = 60 + dag_dir_list_interval = 300 child_process_log_directory = /root/airflow/logs/scheduler + # Task instances listen for external kill signal (when you clear tasks + # from the CLI or the UI), this defines the frequency at which they should + # listen (in seconds). + job_heartbeat_sec = 5 + max_threads = 16 + + # The scheduler constantly tries to trigger new tasks (look at the + # scheduler section in the docs for more information). This defines + # how often the scheduler should run (in seconds). + scheduler_heartbeat_sec = 5 + + # after how much time should the scheduler terminate in seconds + # -1 indicates to run continuously (see also num_runs) + run_duration = -1 + + # after how much time a new DAGs should be picked up from the filesystem + min_file_process_interval = 0 + + statsd_on = False + statsd_host = localhost + statsd_port = 8125 + statsd_prefix = airflow + + # How many seconds to wait between file-parsing loops to prevent the logs from being spammed. + min_file_parsing_loop_time = 1 + + print_stats_interval = 30 + scheduler_zombie_task_threshold = 300 + max_tis_per_query = 0 + authenticate = False [kubernetes] airflow_configmap = airflow-configmap http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a15b7c5b/scripts/ci/kubernetes/minikube/start_minikube.sh ---------------------------------------------------------------------- diff --git a/scripts/ci/kubernetes/minikube/start_minikube.sh b/scripts/ci/kubernetes/minikube/start_minikube.sh index 525529b..7e50c23 100755 --- a/scripts/ci/kubernetes/minikube/start_minikube.sh +++ b/scripts/ci/kubernetes/minikube/start_minikube.sh @@ -15,66 +15,92 @@ # specific language governing permissions and limitations # under the License. -# Guard against a kubernetes cluster already being up -kubectl get pods &> /dev/null -if [ $? -eq 0 ]; then - echo "kubectl get pods returned 0 exit code, exiting early" - exit 0 -fi -# +# This script was based on one made by @kimoonkim for kubernetes-hdfs -curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.26.0/minikube-linux-amd64 && chmod +x minikube -curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl +#!/usr/bin/env bash -sudo mkdir -p /usr/local/bin -sudo mv minikube /usr/local/bin/minikube -sudo mv kubectl /usr/local/bin/kubectl +_MY_SCRIPT="${BASH_SOURCE[0]}" +_MY_DIR=$(cd "$(dirname "$_MY_SCRIPT")" && pwd) +# Avoids 1.7.x because of https://github.com/kubernetes/minikube/issues/2240 +_KUBERNETES_VERSION="${KUBERNETES_VERSION}" + +echo "setting up kubernetes ${_KUBERNETES_VERSION}" + +_MINIKUBE_VERSION="v0.25.2" +_HELM_VERSION=v2.8.1 +_VM_DRIVER=none +USE_MINIKUBE_DRIVER_NONE=true + +_UNAME_OUT=$(uname -s) +case "${_UNAME_OUT}" in + Linux*) _MY_OS=linux;; + Darwin*) _MY_OS=darwin;; + *) _MY_OS="UNKNOWN:${unameOut}" +esac +echo "Local OS is ${_MY_OS}" export MINIKUBE_WANTUPDATENOTIFICATION=false export MINIKUBE_WANTREPORTERRORPROMPT=false -export MINIKUBE_HOME=$HOME export CHANGE_MINIKUBE_NONE_USER=true -mkdir $HOME/.kube || true -touch $HOME/.kube/config -export KUBECONFIG=$HOME/.kube/config +cd $_MY_DIR -start_minikube(){ - sudo -E minikube start --vm-driver=none --kubernetes-version="${KUBERNETES_VERSION}" +rm -rf tmp +mkdir -p bin tmp - # this for loop waits until kubectl can access the api server that minikube has created - for i in {1..90} # timeout 3 minutes - do - echo "------- Running kubectl get pods -------" - STDERR=$(kubectl get pods 2>&1 >/dev/null) - if [ $? -eq 0 ]; then - echo $STDERR +sudo mkdir -p /usr/local/bin - # We do not need dynamic hostpath provisioning, so disable the default storageclass - sudo -E minikube addons disable default-storageclass && kubectl delete storageclasses --all +if [[ ! -x /usr/local/bin/kubectl ]]; then + echo Downloading kubectl, which is a requirement for using minikube. + curl -Lo bin/kubectl \ + https://storage.googleapis.com/kubernetes-release/release/${_KUBERNETES_VERSION}/bin/${_MY_OS}/amd64/kubectl + chmod +x bin/kubectl +fi +if [[ ! -x /usr/local/bin/minikube ]]; then + echo Downloading minikube. + curl -Lo bin/minikube \ + https://storage.googleapis.com/minikube/releases/${_MINIKUBE_VERSION}/minikube-${_MY_OS}-amd64 + chmod +x bin/minikube +fi + +sudo mv bin/minikube /usr/local/bin/minikube +sudo mv bin/kubectl /usr/local/bin/kubectl - # We need to give permission to watch pods to the airflow scheduler. - # The easiest way to do that is by giving admin access to the default serviceaccount (NOT SAFE!) - kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=default:default - exit 0 +export PATH="${_MY_DIR}/bin:$PATH" + +if [[ "${USE_MINIKUBE_DRIVER_NONE:-}" = "true" ]]; then + # Run minikube with none driver. + # See https://blog.travis-ci.com/2017-10-26-running-kubernetes-on-travis-ci-with-minikube + _VM_DRIVER="--vm-driver=none" + if [[ ! -x /usr/local/bin/nsenter ]]; then + # From https://engineering.bitnami.com/articles/implementing-kubernetes-integration-tests-in-travis.html + # Travis ubuntu trusty env doesn't have nsenter, needed for --vm-driver=none + which nsenter >/dev/null && return 0 + echo "INFO: Building 'nsenter' ..." +cat <<-EOF | docker run -i --rm -v "$(pwd):/build" ubuntu:14.04 >& nsenter.build.log + apt-get update + apt-get install -qy git bison build-essential autopoint libtool automake autoconf gettext pkg-config + git clone --depth 1 git://git.kernel.org/pub/scm/utils/util-linux/util-linux.git /tmp/util-linux + cd /tmp/util-linux + ./autogen.sh + ./configure --without-python --disable-all-programs --enable-nsenter + make nsenter + cp -pfv nsenter /build +EOF + if [ ! -f ./nsenter ]; then + echo "ERROR: nsenter build failed, log:" + cat nsenter.build.log + return 1 fi - echo $STDERR - sleep 2 - done -} - -cleanup_minikube(){ - sudo -E minikube stop - sudo -E minikube delete - docker stop $(docker ps -a -q) || true - docker rm $(docker ps -a -q) || true - sleep 1 -} - -start_minikube -echo "Minikube cluster creation timedout. Attempting to restart the minikube cluster." -cleanup_minikube -start_minikube -echo "Minikube cluster creation timedout a second time. Failing." - -exit 1 + echo "INFO: nsenter build OK" + sudo mv ./nsenter /usr/local/bin + fi +fi + +echo "your path is ${PATH}" + +_MINIKUBE="sudo PATH=$PATH minikube" + +$_MINIKUBE config set bootstrapper localkube +$_MINIKUBE start --kubernetes-version=${_KUBERNETES_VERSION} --vm-driver=none +$_MINIKUBE update-context
