Repository: incubator-airflow Updated Branches: refs/heads/master c3c4a8fdc -> 1e36b37b6
[AIRFLOW-1755] Allow mount below root This enables Airflow and Celery Flower to live below root. Draws on the work of Geatan Semet (@Stibbons). This closes #2723 and closes #2818 Closes #2952 from bolkedebruin/AIRFLOW-1755 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/1e36b37b Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/1e36b37b Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/1e36b37b Branch: refs/heads/master Commit: 1e36b37b68ab354d1d7d1d1d3abd151ce2a7cac7 Parents: c3c4a8f Author: Bolke de Bruin <[email protected]> Authored: Fri Jan 19 18:54:26 2018 +0100 Committer: Bolke de Bruin <[email protected]> Committed: Fri Jan 19 18:54:26 2018 +0100 ---------------------------------------------------------------------- airflow/bin/cli.py | 18 +++++-- airflow/config_templates/default_airflow.cfg | 8 +++ .../templates/metastore_browser/base.html | 10 ++-- airflow/www/app.py | 20 +++++-- airflow/www/templates/airflow/dag_details.html | 8 +-- airflow/www/templates/airflow/dags.html | 18 +++---- airflow/www/templates/airflow/graph.html | 8 +-- airflow/www/templates/airflow/list_dags.html | 4 +- airflow/www/templates/airflow/nvd3.html | 8 +-- airflow/www/views.py | 3 ++ docs/integration.rst | 57 ++++++++++++++++++++ setup.py | 1 + tests/www/test_views.py | 22 ++++++++ 13 files changed, 150 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/bin/cli.py ---------------------------------------------------------------------- diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py index e98838d..b032729 100755 --- a/airflow/bin/cli.py +++ b/airflow/bin/cli.py @@ -1054,6 +1054,10 @@ def flower(args): if args.broker_api: api = '--broker_api=' + args.broker_api + url_prefix = '' + if args.url_prefix: + url_prefix = '--url-prefix=' + args.url_prefix + flower_conf = '' if args.flower_conf: flower_conf = '--conf=' + args.flower_conf @@ -1070,7 +1074,8 @@ def flower(args): ) with ctx: - os.execvp("flower", ['flower', '-b', broka, address, port, api, flower_conf]) + os.execvp("flower", ['flower', '-b', + broka, address, port, api, flower_conf, url_prefix]) stdout.close() stderr.close() @@ -1078,7 +1083,8 @@ def flower(args): signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGTERM, sigint_handler) - os.execvp("flower", ['flower', '-b', broka, address, port, api, flower_conf]) + os.execvp("flower", ['flower', '-b', + broka, address, port, api, flower_conf, url_prefix]) def kerberos(args): # noqa @@ -1410,6 +1416,10 @@ class CLIFactory(object): 'flower_conf': Arg( ("-fc", "--flower_conf"), help="Configuration file for flower"), + 'flower_url_prefix': Arg( + ("-u", "--url_prefix"), + default=conf.get('celery', 'FLOWER_URL_PREFIX'), + help="URL prefix for Flower"), 'task_params': Arg( ("-tp", "--task_params"), help="Sends a JSON params dict to the task"), @@ -1584,8 +1594,8 @@ class CLIFactory(object): }, { 'func': flower, 'help': "Start a Celery Flower", - 'args': ('flower_hostname', 'flower_port', 'flower_conf', 'broker_api', - 'pid', 'daemon', 'stdout', 'stderr', 'log_file'), + 'args': ('flower_hostname', 'flower_port', 'flower_conf', 'flower_url_prefix', + 'broker_api', 'pid', 'daemon', 'stdout', 'stderr', 'log_file'), }, { 'func': version, 'help': "Show the version", http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/config_templates/default_airflow.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index d0dfb72..59ff740 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -155,6 +155,10 @@ killed_task_cleanup_time = 60 # database directly, while the json_client will use the api running on the # webserver api_client = airflow.api.client.local_client + +# If you set web_server_url_prefix, do NOT forget to append it here, ex: +# endpoint_url = http://localhost:8080/myroot +# So api will look like: http://localhost:8080/myroot/api/experimental/... endpoint_url = http://localhost:8080 [api] @@ -312,6 +316,10 @@ result_backend = db+mysql://airflow:airflow@localhost:3306/airflow # it `airflow flower`. This defines the IP that Celery Flower runs on flower_host = 0.0.0.0 +# The root URL for Flower +# Ex: flower_url_prefix = /flower +flower_url_prefix = + # This defines the port that Celery Flower runs on flower_port = 5555 http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html ---------------------------------------------------------------------- diff --git a/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html b/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html index ba1e7e2..26675bb 100644 --- a/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html +++ b/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html @@ -1,13 +1,13 @@ -{# +{# Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,14 +31,14 @@ {% block head %} {{ super() }} <link rel="stylesheet" type="text/css" href="{{ url_for("static", filename="dataTables.bootstrap.css") }}"> -<link href="/admin/static/vendor/select2/select2.css" rel="stylesheet"> +<link href="{{ url_for('static', filename='select2.css') }}" rel="stylesheet"> {% endblock %} {% block tail %} {{ super() }} <script src="{{ url_for('static', filename='jquery.dataTables.min.js') }}"></script> <script src="{{ url_for('static', filename='dataTables.bootstrap.js') }}"></script> -<script src="/admin/static/vendor/select2/select2.min.js" type="text/javascript"></script> +<script src="{{ url_for('static', filename='select2.min.js') }}" type="text/javascript"></script> <script> // Filling up the table selector url = "{{ url_for('.objects') }}"; http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/app.py ---------------------------------------------------------------------- diff --git a/airflow/www/app.py b/airflow/www/app.py index 0b71c17..d46935b 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -19,7 +19,8 @@ from flask import Flask from flask_admin import Admin, base from flask_caching import Cache from flask_wtf.csrf import CSRFProtect -csrf = CSRFProtect() +from six.moves.urllib.parse import urlparse +from werkzeug.wsgi import DispatcherMiddleware import airflow from airflow import configuration as conf @@ -32,6 +33,8 @@ from airflow import jobs from airflow import settings from airflow import configuration +csrf = CSRFProtect() + def create_app(config=None, testing=False): app = Flask(__name__) @@ -154,11 +157,22 @@ def create_app(config=None, testing=False): return app + app = None -def cached_app(config=None): +def root_app(env, resp): + resp(b'404 Not Found', [(b'Content-Type', b'text/plain')]) + return [b'Apache Airflow is not at this location'] + + +def cached_app(config=None, testing=False): global app if not app: - app = create_app(config) + base_url = urlparse(configuration.get('webserver', 'base_url'))[2] + if not base_url or base_url == '/': + base_url = "" + + app = create_app(config, testing) + app = DispatcherMiddleware(root_app, {base_url: app}) return app http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/dag_details.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/dag_details.html b/airflow/www/templates/airflow/dag_details.html index 716932a..4f9f2eb 100644 --- a/airflow/www/templates/airflow/dag_details.html +++ b/airflow/www/templates/airflow/dag_details.html @@ -1,13 +1,13 @@ -{# +{# Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,7 +29,7 @@ <a class="btn" style="border: none; background-color:{{ State.color(state)}}; color: {{ State.color_fg(state) }};" - href="/admin/taskinstance/?flt0_dag_id_equals={{ dag.dag_id }}&flt2_state_equals={{ state }}"> + href="{{ url_for('taskinstance.index_view') }}?flt0_dag_id_equals={{ dag.dag_id }}&flt2_state_equals={{ state }}"> {{ state }} <span class="badge">{{ count }}</span> </a> {% endfor %} http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/dags.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html index eeef790..6b051c3 100644 --- a/airflow/www/templates/airflow/dags.html +++ b/airflow/www/templates/airflow/dags.html @@ -69,7 +69,7 @@ <!-- Column 1: Edit dag --> <td class="text-center" style="width:10px;"> {% if dag_id in orm_dags %} - <a href="/admin/dagmodel/edit/?id={{ dag_id }}" title="Info"> + <a href="{{ url_for('dagmodel.edit_view') }}?id={{ dag_id }}" title="Info"> <span class="glyphicon glyphicon-edit" aria-hidden="true"></span> </a> {% endif %} @@ -100,7 +100,7 @@ <!-- Column 4: Dag Schedule --> <td> {% if dag_id in webserver_dags %} - <a class="label label-default schedule {{ dag.dag_id }}" href="/admin/dagrun/?flt2_dag_id_equals={{ dag.dag_id }}"> + <a class="label label-default schedule {{ dag.dag_id }}" href="{{ url_for('dagrun.index_view') }}?flt2_dag_id_equals={{ dag.dag_id }}"> {{ dag.schedule_interval }} </a> {% endif %} @@ -180,7 +180,7 @@ </a> <!-- Logs --> - <a href="/admin/log/?sort=1&desc=1&flt1_dag_id_equals={{ dag.dag_id }}"> + <a href="{{ url_for('log.index_view') }}?sort=1&desc=1&flt1_dag_id_equals={{ dag.dag_id }}"> <span class="glyphicon glyphicon-align-justify" aria-hidden="true" data-original-title="Logs"></span> </a> {% endif %} @@ -209,9 +209,9 @@ </div> {% if not hide_paused %} - <a href="/admin/?showPaused=False">Hide Paused DAGs</a> + <a href="{{ url_for('admin.index') }}?showPaused=False">Hide Paused DAGs</a> {% else %} - <a href="/admin/?showPaused=True">Show Paused DAGs</a> + <a href="{{ url_for('admin.index') }}?showPaused=True">Show Paused DAGs</a> {% endif %} </div> {% endblock %} @@ -224,8 +224,8 @@ <script src="{{ url_for('static', filename='bootstrap3-typeahead.min.js') }}"></script> <script> - const DAGS_INDEX = {{ url_for('admin.index') }} - const ENTER_KEY_CODE = 13 + const DAGS_INDEX = "{{ url_for('admin.index') }}"; + const ENTER_KEY_CODE = 13; $('#dag_query').on('keypress', function (e) { // check for key press on ENTER (key code 13) to trigger the search @@ -353,7 +353,7 @@ }) .on('click', function(d, i) { if (d.count > 0) - window.location = "/admin/dagrun/?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state; + window.location = "{{ url_for('dagrun.index_view') }}?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state; }) .on('mouseover', function(d, i) { if (d.count > 0) { @@ -432,7 +432,7 @@ }) .on('click', function(d, i) { if (d.count > 0) - window.location = "/admin/taskinstance/?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state; + window.location = "{{ url_for('taskinstance.index_view') }}?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state; }) .on('mouseover', function(d, i) { if (d.count > 0) { http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/graph.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/graph.html b/airflow/www/templates/airflow/graph.html index 24fc508..e17c89b 100644 --- a/airflow/www/templates/airflow/graph.html +++ b/airflow/www/templates/airflow/graph.html @@ -1,13 +1,13 @@ -{# +{# Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -350,7 +350,7 @@ $("#loading").css("display", "block"); $("div#svg_container").css("opacity", "0.2"); $.get( - "/admin/airflow/object/task_instances", + "{{ url_for('airflow.task_instances') }}", {dag_id : "{{ dag.dag_id }}", execution_date : "{{ execution_date }}"}) .done( function(task_instances) { http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/list_dags.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/list_dags.html b/airflow/www/templates/airflow/list_dags.html index 9ace2fd..e8533d7 100644 --- a/airflow/www/templates/airflow/list_dags.html +++ b/airflow/www/templates/airflow/list_dags.html @@ -172,7 +172,7 @@ <a href="{{ url_for("airflow.refresh", dag_id=row.dag_id) }}" title="Refresh"> <span class="glyphicon glyphicon-refresh" aria-hidden="true"></span> </a> - <a href="/admin/log/?sort=1&desc=1&flt1_dag_id_equals={{ row.dag_id }}" title="Logs"> + <a href="{{ url_for('log.index_view') }}?sort=1&desc=1&flt1_dag_id_equals={{ row.dag_id }}" title="Logs"> <i class="icon-list"></i> <span class="glyphicon glyphicon-align-justify" aria-hidden="true"></span> </a> @@ -276,7 +276,7 @@ }) .on('click', function(d, i) { if (d.count > 0) - window.location = "/admin/taskinstance/?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state; + window.location = "{{ url_for('taskinstance.index_view') }}?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state; }) .on('mouseover', function(d, i) { if (d.count > 0) { http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/nvd3.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/nvd3.html b/airflow/www/templates/airflow/nvd3.html index 5478ff8..45aca95 100644 --- a/airflow/www/templates/airflow/nvd3.html +++ b/airflow/www/templates/airflow/nvd3.html @@ -1,13 +1,13 @@ -{# +{# Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -71,7 +71,7 @@ body { <div id="container"> <h2> <span id="label">{{ label }}</span> - <a href="/admin/chart/edit/?id={{ chart.id }}" > + <a href="{{ url_for('chart.edit_view') }}?id={{ chart.id }}" > <span class="glyphicon glyphicon-edit" aria-hidden="true" ></span> </a> </h2> http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/views.py ---------------------------------------------------------------------- diff --git a/airflow/www/views.py b/airflow/www/views.py index cc73c8b..252241a 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -108,6 +108,9 @@ if conf.getboolean('webserver', 'FILTER_BY_OWNER'): def dag_link(v, c, m, p): + if m.dag_id is None: + return Markup() + dag_id = bleach.clean(m.dag_id) url = url_for( 'airflow.graph', http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/docs/integration.rst ---------------------------------------------------------------------- diff --git a/docs/integration.rst b/docs/integration.rst index 142fca2..734ecad 100644 --- a/docs/integration.rst +++ b/docs/integration.rst @@ -1,12 +1,69 @@ Integration =========== +- :ref:`ReverseProxy` - :ref:`Azure` - :ref:`AWS` - :ref:`Databricks` - :ref:`GCP` +.. _ReverseProxy: +Reverse Proxy +------------- +Airflow can be set up behind a reverse proxy, with the ability to set its endpoint with great +flexibility. + +For example, you can configure your reverse proxy to get: + +:: + + https://lab.mycompany.com/myorg/airflow/ + +To do so, you need to set the following setting in your `airflow.cfg`:: + + base_url = http://my_host/myorg/airflow + +Additionally if you use Celery Executor, you can get Flower in `/myorg/flower` with:: + + flower_url_prefix = /myorg/flower + +Your reverse proxy (ex: nginx) should be configured as follow: + +- pass the url and http header as it for the Airflow webserver, without any rewrite, for example:: + + server { + listen 80; + server_name lab.mycompany.com; + + location /myorg/airflow/ { + proxy_pass http://localhost:8080; + proxy_set_header Host $host; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + } + +- rewrite the url for the flower endpoint:: + + server { + listen 80; + server_name lab.mycompany.com; + + location /myorg/flower/ { + rewrite ^/myorg/flower/(.*)$ /$1 break; # remove prefix from http header + proxy_pass http://localhost:5555; + proxy_set_header Host $host; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + } + + .. _Azure: Azure: Microsoft Azure http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 9358090..97fafe8 100644 --- a/setup.py +++ b/setup.py @@ -234,6 +234,7 @@ def do_setup(): 'tabulate>=0.7.5, <0.8.0', 'thrift>=0.9.2', 'tzlocal>=1.4', + 'werkzeug>=0.14.1, <0.15.0', 'zope.deprecation>=4.0, <5.0', ], setup_requires=[ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/tests/www/test_views.py ---------------------------------------------------------------------- diff --git a/tests/www/test_views.py b/tests/www/test_views.py index 6ea8db2..ff20333 100644 --- a/tests/www/test_views.py +++ b/tests/www/test_views.py @@ -21,6 +21,8 @@ import tempfile import unittest import sys +from werkzeug.test import Client + from airflow import models, configuration, settings from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.models import DAG, TaskInstance @@ -424,5 +426,25 @@ class TestVarImportView(unittest.TestCase): self.assertIn('VALUE', body) +class TestMountPoint(unittest.TestCase): + def setUp(self): + super(TestMountPoint, self).setUp() + configuration.load_test_config() + configuration.conf.set("webserver", "base_url", "http://localhost:8080/test";) + config = dict() + config['WTF_CSRF_METHODS'] = [] + app = application.cached_app(config=config, testing=True) + self.client = Client(app) + + def test_mount(self): + response, _, _ = self.client.get('/', follow_redirects=True) + txt = b''.join(response) + self.assertEqual(b"Apache Airflow is not at this location", txt) + + response, _, _ = self.client.get('/test', follow_redirects=True) + resp_html = b''.join(response) + self.assertIn(b"DAGs", resp_html) + + if __name__ == '__main__': unittest.main()
