Repository: incubator-airflow Updated Branches: refs/heads/master 2fb84a89b -> ef7fe870e
[AIRFLOW-1863][AIRFLOW-2529] Add dag run selection widgets to gantt view Add same widgets to filter and select dag run known from graph view to the gantt chart view. Extract common code to handle request parameters and DB query. Closes #3450 from seelmann/AIRFLOW-1863-gantt-view Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/ef7fe870 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/ef7fe870 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/ef7fe870 Branch: refs/heads/master Commit: ef7fe870e0a77398f1579eeae027da6cda083704 Parents: 2fb84a8 Author: Stefan Seelmann <[email protected]> Authored: Fri Jun 1 18:59:22 2018 -0700 Committer: r39132 <[email protected]> Committed: Fri Jun 1 18:59:22 2018 -0700 ---------------------------------------------------------------------- airflow/www/forms.py | 6 +- airflow/www/templates/airflow/gantt.html | 2 + airflow/www/views.py | 120 +++++++------- airflow/www_rbac/forms.py | 7 +- airflow/www_rbac/templates/airflow/gantt.html | 2 + airflow/www_rbac/views.py | 119 +++++++------- tests/www/test_views.py | 151 +++++++++++++----- tests/www_rbac/test_views.py | 177 ++++++++++++++------- 8 files changed, 377 insertions(+), 207 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ef7fe870/airflow/www/forms.py ---------------------------------------------------------------------- diff --git a/airflow/www/forms.py b/airflow/www/forms.py index d6fbe13..42bf470 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -29,7 +29,7 @@ from flask_wtf import FlaskForm class DateTimeForm(FlaskForm): - # Date filter form needed for gantt and graph view + # Date filter form for task views execution_date = DateTimeField( "Execution date", widget=DateTimePickerWidget()) @@ -47,3 +47,7 @@ class DateTimeWithNumRunsForm(FlaskForm): (365, "365"), )) + +class DateTimeWithNumRunsWithDagRunsForm(DateTimeWithNumRunsForm): + # Date time and number of runs and dag runs form for graph and gantt view + execution_date = SelectField("DAG run") http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ef7fe870/airflow/www/templates/airflow/gantt.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/gantt.html b/airflow/www/templates/airflow/gantt.html index ecf9b6d..01d3cd8 100644 --- a/airflow/www/templates/airflow/gantt.html +++ b/airflow/www/templates/airflow/gantt.html @@ -28,6 +28,8 @@ {{ super() }} <form method="get"> <div class="form-inline"> + Base date: {{ form.base_date(class_="form-control") }} + Number of runs: {{ form.num_runs(class_="form-control") }} Run:<input type="hidden" value="{{ dag.dag_id }}" name="dag_id"> {{ form.execution_date(class_="form-control") | safe }} <input type="submit" value="Go" class="btn btn-default" action="" method="get"> http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ef7fe870/airflow/www/views.py ---------------------------------------------------------------------- diff --git a/airflow/www/views.py b/airflow/www/views.py index 3d09670..a16b685 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -89,7 +89,8 @@ from airflow.utils.dates import infer_time_unit, scale_time_units, parse_executi from airflow.utils.timezone import datetime from airflow.utils.net import get_hostname from airflow.www import utils as wwwutils -from airflow.www.forms import DateTimeForm, DateTimeWithNumRunsForm +from airflow.www.forms import (DateTimeForm, DateTimeWithNumRunsForm, + DateTimeWithNumRunsWithDagRunsForm) from airflow.www.validators import GreaterEqualThan QUERY_LIMIT = 100000 @@ -318,6 +319,57 @@ def get_chart_height(dag): return 600 + len(dag.tasks) * 10 +def get_date_time_num_runs_dag_runs_form_data(request, session, dag): + dttm = request.args.get('execution_date') + if dttm: + dttm = pendulum.parse(dttm) + else: + dttm = dag.latest_execution_date or timezone.utcnow() + + base_date = request.args.get('base_date') + if base_date: + base_date = timezone.parse(base_date) + else: + # The DateTimeField widget truncates milliseconds and would loose + # the first dag run. Round to next second. + base_date = (dttm + timedelta(seconds=1)).replace(microsecond=0) + + default_dag_run = conf.getint('webserver', 'default_dag_run_display_number') + num_runs = request.args.get('num_runs') + num_runs = int(num_runs) if num_runs else default_dag_run + + DR = models.DagRun + drs = ( + session.query(DR) + .filter( + DR.dag_id == dag.dag_id, + DR.execution_date <= base_date) + .order_by(desc(DR.execution_date)) + .limit(num_runs) + .all() + ) + dr_choices = [] + dr_state = None + for dr in drs: + dr_choices.append((dr.execution_date.isoformat(), dr.run_id)) + if dttm == dr.execution_date: + dr_state = dr.state + + # Happens if base_date was changed and the selected dag run is not in result + if not dr_state and drs: + dr = drs[0] + dttm = dr.execution_date + dr_state = dr.state + + return { + 'dttm': dttm, + 'base_date': base_date, + 'num_runs': num_runs, + 'execution_date': dttm.isoformat(), + 'dr_choices': dr_choices, + 'dr_state': dr_state, + } + class Airflow(BaseView): def is_visible(self): return False @@ -1356,7 +1408,6 @@ class Airflow(BaseView): @wwwutils.action_logging @provide_session def graph(self, session=None): - default_dag_run = conf.getint('webserver', 'default_dag_run_display_number') dag_id = request.args.get('dag_id') blur = conf.getboolean('webserver', 'demo_mode') dag = dagbag.get_dag(dag_id) @@ -1398,48 +1449,11 @@ class Airflow(BaseView): for t in dag.roots: get_upstream(t) - dttm = request.args.get('execution_date') - if dttm: - dttm = pendulum.parse(dttm) - else: - dttm = dag.latest_execution_date or timezone.utcnow() - - base_date = request.args.get('base_date') - if base_date: - base_date = timezone.parse(base_date) - else: - # The DateTimeField widget truncates milliseconds and would loose - # the first dag run. Round to next second. - base_date = (dttm + timedelta(seconds=1)).replace(microsecond=0) - - num_runs = request.args.get('num_runs') - num_runs = int(num_runs) if num_runs else default_dag_run + dt_nr_dr_data = get_date_time_num_runs_dag_runs_form_data(request, session, dag) + dt_nr_dr_data['arrange'] = arrange + dttm = dt_nr_dr_data['dttm'] - DR = models.DagRun - drs = ( - session.query(DR) - .filter( - DR.dag_id == dag.dag_id, - DR.execution_date <= base_date) - .order_by(desc(DR.execution_date)) - .limit(num_runs) - .all() - ) - dr_choices = [] - dr_state = None - for dr in drs: - dr_choices.append((dr.execution_date.isoformat(), dr.run_id)) - if dttm == dr.execution_date: - dr_state = dr.state - - # Happens if base_date was changed and the selected dag run is not in result - if not dr_state and drs: - dr = drs[0] - dttm = dr.execution_date - dr_state = dr.state - - class GraphForm(DateTimeWithNumRunsForm): - execution_date = SelectField("DAG run", choices=dr_choices) + class GraphForm(DateTimeWithNumRunsWithDagRunsForm): arrange = SelectField("Layout", choices=( ('LR', "Left->Right"), ('RL', "Right->Left"), @@ -1447,11 +1461,8 @@ class Airflow(BaseView): ('BT', "Bottom->Top"), )) - form = GraphForm( - data={'execution_date': dttm.isoformat(), - 'arrange': arrange, - 'base_date': base_date, - 'num_runs': num_runs}) + form = GraphForm(data=dt_nr_dr_data) + form.execution_date.choices = dt_nr_dr_data['dr_choices'] task_instances = { ti.task_id: alchemy_to_dict(ti) @@ -1474,7 +1485,7 @@ class Airflow(BaseView): width=request.args.get('width', "100%"), height=request.args.get('height', "800"), execution_date=dttm.isoformat(), - state_token=state_token(dr_state), + state_token=state_token(dt_nr_dr_data['dr_state']), doc_md=doc_md, arrange=arrange, operators=sorted( @@ -1789,7 +1800,6 @@ class Airflow(BaseView): @wwwutils.action_logging @provide_session def gantt(self, session=None): - dag_id = request.args.get('dag_id') dag = dagbag.get_dag(dag_id) demo_mode = conf.getboolean('webserver', 'demo_mode') @@ -1801,13 +1811,11 @@ class Airflow(BaseView): include_upstream=True, include_downstream=False) - dttm = request.args.get('execution_date') - if dttm: - dttm = pendulum.parse(dttm) - else: - dttm = dag.latest_execution_date or timezone.utcnow() + dt_nr_dr_data = get_date_time_num_runs_dag_runs_form_data(request, session, dag) + dttm = dt_nr_dr_data['dttm'] - form = DateTimeForm(data={'execution_date': dttm}) + form = DateTimeWithNumRunsWithDagRunsForm(data=dt_nr_dr_data) + form.execution_date.choices = dt_nr_dr_data['dr_choices'] tis = [ ti for ti in dag.get_task_instances(session, dttm, dttm) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ef7fe870/airflow/www_rbac/forms.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/forms.py b/airflow/www_rbac/forms.py index 61a7ce2..da9d12c 100644 --- a/airflow/www_rbac/forms.py +++ b/airflow/www_rbac/forms.py @@ -38,7 +38,7 @@ from wtforms.fields import (IntegerField, SelectField, TextAreaField, PasswordFi class DateTimeForm(FlaskForm): - # Date filter form needed for gantt and graph view + # Date filter form needed for task views execution_date = DateTimeField( "Execution date", widget=DateTimePickerWidget()) @@ -57,6 +57,11 @@ class DateTimeWithNumRunsForm(FlaskForm): )) +class DateTimeWithNumRunsWithDagRunsForm(DateTimeWithNumRunsForm): + # Date time and number of runs and dag runs form for graph and gantt view + execution_date = SelectField("DAG run") + + class DagRunForm(DynamicForm): dag_id = StringField( lazy_gettext('Dag Id'), http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ef7fe870/airflow/www_rbac/templates/airflow/gantt.html ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/templates/airflow/gantt.html b/airflow/www_rbac/templates/airflow/gantt.html index 3cdb269..3a567ee 100644 --- a/airflow/www_rbac/templates/airflow/gantt.html +++ b/airflow/www_rbac/templates/airflow/gantt.html @@ -29,6 +29,8 @@ {{ super() }} <form method="get"> <div class="form-inline"> + Base date: {{ form.base_date(class_="form-control") }} + Number of runs: {{ form.num_runs(class_="form-control") }} Run:<input type="hidden" value="{{ dag.dag_id }}" name="dag_id"> {{ form.execution_date(class_="form-control") | safe }} <input type="submit" value="Go" class="btn btn-default" action="" method="get"> http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ef7fe870/airflow/www_rbac/views.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/views.py b/airflow/www_rbac/views.py index e09c384..3a1951a 100644 --- a/airflow/www_rbac/views.py +++ b/airflow/www_rbac/views.py @@ -72,6 +72,7 @@ from airflow.www_rbac import utils as wwwutils from airflow.www_rbac.app import app from airflow.www_rbac.decorators import action_logging, gzipped from airflow.www_rbac.forms import (DateTimeForm, DateTimeWithNumRunsForm, + DateTimeWithNumRunsWithDagRunsForm, DagRunForm, ConnectionForm) from airflow.www_rbac.security import is_view_only from airflow.www_rbac.widgets import AirflowModelListWidget @@ -79,6 +80,59 @@ from airflow.www_rbac.widgets import AirflowModelListWidget PAGE_SIZE = conf.getint('webserver', 'page_size') dagbag = models.DagBag(settings.DAGS_FOLDER) + +def get_date_time_num_runs_dag_runs_form_data(request, session, dag): + dttm = request.args.get('execution_date') + if dttm: + dttm = pendulum.parse(dttm) + else: + dttm = dag.latest_execution_date or timezone.utcnow() + + base_date = request.args.get('base_date') + if base_date: + base_date = timezone.parse(base_date) + else: + # The DateTimeField widget truncates milliseconds and would loose + # the first dag run. Round to next second. + base_date = (dttm + timedelta(seconds=1)).replace(microsecond=0) + + default_dag_run = conf.getint('webserver', 'default_dag_run_display_number') + num_runs = request.args.get('num_runs') + num_runs = int(num_runs) if num_runs else default_dag_run + + DR = models.DagRun + drs = ( + session.query(DR) + .filter( + DR.dag_id == dag.dag_id, + DR.execution_date <= base_date) + .order_by(desc(DR.execution_date)) + .limit(num_runs) + .all() + ) + dr_choices = [] + dr_state = None + for dr in drs: + dr_choices.append((dr.execution_date.isoformat(), dr.run_id)) + if dttm == dr.execution_date: + dr_state = dr.state + + # Happens if base_date was changed and the selected dag run is not in result + if not dr_state and drs: + dr = drs[0] + dttm = dr.execution_date + dr_state = dr.state + + return { + 'dttm': dttm, + 'base_date': base_date, + 'num_runs': num_runs, + 'execution_date': dttm.isoformat(), + 'dr_choices': dr_choices, + 'dr_state': dr_state, + } + + ###################################################################################### # BaseViews ###################################################################################### @@ -1031,7 +1085,6 @@ class Airflow(AirflowBaseView): @action_logging @provide_session def graph(self, session=None): - default_dag_run = conf.getint('webserver', 'default_dag_run_display_number') dag_id = request.args.get('dag_id') blur = conf.getboolean('webserver', 'demo_mode') dag = dagbag.get_dag(dag_id) @@ -1073,48 +1126,11 @@ class Airflow(AirflowBaseView): for t in dag.roots: get_upstream(t) - dttm = request.args.get('execution_date') - if dttm: - dttm = pendulum.parse(dttm) - else: - dttm = dag.latest_execution_date or timezone.utcnow() + dt_nr_dr_data = get_date_time_num_runs_dag_runs_form_data(request, session, dag) + dt_nr_dr_data['arrange'] = arrange + dttm = dt_nr_dr_data['dttm'] - base_date = request.args.get('base_date') - if base_date: - base_date = timezone.parse(base_date) - else: - # The DateTimeField widget truncates milliseconds and would loose - # the first dag run. Round to next second. - base_date = (dttm + timedelta(seconds=1)).replace(microsecond=0) - - num_runs = request.args.get('num_runs') - num_runs = int(num_runs) if num_runs else default_dag_run - - DR = models.DagRun - drs = ( - session.query(DR) - .filter( - DR.dag_id == dag.dag_id, - DR.execution_date <= base_date) - .order_by(desc(DR.execution_date)) - .limit(num_runs) - .all() - ) - dr_choices = [] - dr_state = None - for dr in drs: - dr_choices.append((dr.execution_date.isoformat(), dr.run_id)) - if dttm == dr.execution_date: - dr_state = dr.state - - # Happens if base_date was changed and the selected dag run is not in result - if not dr_state and drs: - dr = drs[0] - dttm = dr.execution_date - dr_state = dr.state - - class GraphForm(DateTimeWithNumRunsForm): - execution_date = SelectField("DAG run", choices=dr_choices) + class GraphForm(DateTimeWithNumRunsWithDagRunsForm): arrange = SelectField("Layout", choices=( ('LR', "Left->Right"), ('RL', "Right->Left"), @@ -1122,11 +1138,8 @@ class Airflow(AirflowBaseView): ('BT', "Bottom->Top"), )) - form = GraphForm( - data={'execution_date': dttm.isoformat(), - 'arrange': arrange, - 'base_date': base_date, - 'num_runs': num_runs}) + form = GraphForm(data=dt_nr_dr_data) + form.execution_date.choices = dt_nr_dr_data['dr_choices'] task_instances = { ti.task_id: alchemy_to_dict(ti) @@ -1150,7 +1163,7 @@ class Airflow(AirflowBaseView): width=request.args.get('width', "100%"), height=request.args.get('height', "800"), execution_date=dttm.isoformat(), - state_token=wwwutils.state_token(dr_state), + state_token=wwwutils.state_token(dt_nr_dr_data['dr_state']), doc_md=doc_md, arrange=arrange, operators=sorted( @@ -1477,13 +1490,11 @@ class Airflow(AirflowBaseView): include_upstream=True, include_downstream=False) - dttm = request.args.get('execution_date') - if dttm: - dttm = pendulum.parse(dttm) - else: - dttm = dag.latest_execution_date or timezone.utcnow() + dt_nr_dr_data = get_date_time_num_runs_dag_runs_form_data(request, session, dag) + dttm = dt_nr_dr_data['dttm'] - form = DateTimeForm(data={'execution_date': dttm}) + form = DateTimeWithNumRunsWithDagRunsForm(data=dt_nr_dr_data) + form.execution_date.choices = dt_nr_dr_data['dr_choices'] tis = [ ti for ti in dag.get_task_instances(session, dttm, dttm) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ef7fe870/tests/www/test_views.py ---------------------------------------------------------------------- diff --git a/tests/www/test_views.py b/tests/www/test_views.py index b115c7c..f59470e 100644 --- a/tests/www/test_views.py +++ b/tests/www/test_views.py @@ -504,25 +504,21 @@ class TestMountPoint(unittest.TestCase): self.assertIn(b"DAGs", resp_html) -class TestGraphView(unittest.TestCase): - DAG_ID = 'dag_for_testing_graph_view' +class ViewWithDateTimeAndNumRunsAndDagRunsFormTester: + DAG_ID = 'dag_for_testing_dt_nr_dr_form' DEFAULT_DATE = datetime(2017, 9, 1) RUNS_DATA = [ - ('dag_run_for_testing_graph_view_4', datetime(2018, 4, 4)), - ('dag_run_for_testing_graph_view_3', datetime(2018, 3, 3)), - ('dag_run_for_testing_graph_view_2', datetime(2018, 2, 2)), - ('dag_run_for_testing_graph_view_1', datetime(2018, 1, 1)), + ('dag_run_for_testing_dt_nr_dr_form_4', datetime(2018, 4, 4)), + ('dag_run_for_testing_dt_nr_dr_form_3', datetime(2018, 3, 3)), + ('dag_run_for_testing_dt_nr_dr_form_2', datetime(2018, 2, 2)), + ('dag_run_for_testing_dt_nr_dr_form_1', datetime(2018, 1, 1)), ] - GRAPH_ENDPOINT = '/admin/airflow/graph?dag_id={dag_id}'.format( - dag_id=DAG_ID - ) - @classmethod - def setUpClass(cls): - super(TestGraphView, cls).setUpClass() + def __init__(self, test, endpoint): + self.test = test + self.endpoint = endpoint def setUp(self): - super(TestGraphView, self).setUp() configuration.load_test_config() app = application.create_app(testing=True) app.config['WTF_CSRF_METHODS'] = [] @@ -547,30 +543,25 @@ class TestGraphView(unittest.TestCase): DagRun.dag_id == self.DAG_ID).delete() self.session.commit() self.session.close() - super(TestGraphView, self).tearDown() - - @classmethod - def tearDownClass(cls): - super(TestGraphView, cls).tearDownClass() def assertBaseDateAndNumRuns(self, base_date, num_runs, data): - self.assertNotIn('name="base_date" value="{}"'.format(base_date), data) - self.assertNotIn('<option selected="" value="{}">{}</option>'.format( + self.test.assertNotIn('name="base_date" value="{}"'.format(base_date), data) + self.test.assertNotIn('<option selected="" value="{}">{}</option>'.format( num_runs, num_runs), data) def assertRunIsNotInDropdown(self, run, data): - self.assertNotIn(run.execution_date.isoformat(), data) - self.assertNotIn(run.run_id, data) + self.test.assertNotIn(run.execution_date.isoformat(), data) + self.test.assertNotIn(run.run_id, data) def assertRunIsInDropdownNotSelected(self, run, data): - self.assertIn('<option value="{}">{}</option>'.format( + self.test.assertIn('<option value="{}">{}</option>'.format( run.execution_date.isoformat(), run.run_id), data) def assertRunIsSelected(self, run, data): - self.assertIn('<option selected value="{}">{}</option>'.format( + self.test.assertIn('<option selected value="{}">{}</option>'.format( run.execution_date.isoformat(), run.run_id), data) - def test_graph_view_default_parameters(self): + def test_with_default_parameters(self): """ Tests graph view with no URL parameter. Should show all dag runs in the drop down. @@ -578,18 +569,18 @@ class TestGraphView(unittest.TestCase): Should set base date to current date (not asserted) """ response = self.app.get( - self.GRAPH_ENDPOINT + self.endpoint ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') - self.assertIn('Base date:', data) - self.assertIn('Number of runs:', data) + self.test.assertIn('Base date:', data) + self.test.assertIn('Number of runs:', data) self.assertRunIsSelected(self.runs[0], data) self.assertRunIsInDropdownNotSelected(self.runs[1], data) self.assertRunIsInDropdownNotSelected(self.runs[2], data) self.assertRunIsInDropdownNotSelected(self.runs[3], data) - def test_graph_view_with_execution_date_parameter_only(self): + def test_with_execution_date_parameter_only(self): """ Tests graph view with execution_date URL parameter. Scenario: click link from dag runs view. @@ -598,10 +589,10 @@ class TestGraphView(unittest.TestCase): Should set base date to execution date. """ response = self.app.get( - self.GRAPH_ENDPOINT + '&execution_date={}'.format( + self.endpoint + '&execution_date={}'.format( self.runs[1].execution_date.isoformat()) ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') self.assertBaseDateAndNumRuns( self.runs[1].execution_date, @@ -612,7 +603,7 @@ class TestGraphView(unittest.TestCase): self.assertRunIsInDropdownNotSelected(self.runs[2], data) self.assertRunIsInDropdownNotSelected(self.runs[3], data) - def test_graph_view_with_base_date_and_num_runs_parmeters_only(self): + def test_with_base_date_and_num_runs_parmeters_only(self): """ Tests graph view with base_date and num_runs URL parameters. Should only show dag runs older than base_date in the drop down, @@ -621,10 +612,10 @@ class TestGraphView(unittest.TestCase): Should set base date and num runs to submitted values. """ response = self.app.get( - self.GRAPH_ENDPOINT + '&base_date={}&num_runs=2'.format( + self.endpoint + '&base_date={}&num_runs=2'.format( self.runs[1].execution_date.isoformat()) ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') self.assertBaseDateAndNumRuns(self.runs[1].execution_date, 2, data) self.assertRunIsNotInDropdown(self.runs[0], data) @@ -632,7 +623,7 @@ class TestGraphView(unittest.TestCase): self.assertRunIsInDropdownNotSelected(self.runs[2], data) self.assertRunIsNotInDropdown(self.runs[3], data) - def test_graph_view_with_base_date_and_num_runs_and_execution_date_outside(self): + def test_with_base_date_and_num_runs_and_execution_date_outside(self): """ Tests graph view with base_date and num_runs and execution-date URL parameters. Scenario: change the base date and num runs and press "Go", @@ -642,11 +633,11 @@ class TestGraphView(unittest.TestCase): Should set base date and num runs to submitted values. """ response = self.app.get( - self.GRAPH_ENDPOINT + '&base_date={}&num_runs=42&execution_date={}'.format( + self.endpoint + '&base_date={}&num_runs=42&execution_date={}'.format( self.runs[1].execution_date.isoformat(), self.runs[0].execution_date.isoformat()) ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') self.assertBaseDateAndNumRuns(self.runs[1].execution_date, 42, data) self.assertRunIsNotInDropdown(self.runs[0], data) @@ -654,7 +645,7 @@ class TestGraphView(unittest.TestCase): self.assertRunIsInDropdownNotSelected(self.runs[2], data) self.assertRunIsInDropdownNotSelected(self.runs[3], data) - def test_graph_view_with_base_date_and_num_runs_and_execution_date_within(self): + def test_with_base_date_and_num_runs_and_execution_date_within(self): """ Tests graph view with base_date and num_runs and execution-date URL parameters. Scenario: change the base date and num runs and press "Go", @@ -664,11 +655,11 @@ class TestGraphView(unittest.TestCase): Should set base date and num runs to submitted values. """ response = self.app.get( - self.GRAPH_ENDPOINT + '&base_date={}&num_runs=5&execution_date={}'.format( + self.endpoint + '&base_date={}&num_runs=5&execution_date={}'.format( self.runs[2].execution_date.isoformat(), self.runs[3].execution_date.isoformat()) ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') self.assertBaseDateAndNumRuns(self.runs[2].execution_date, 5, data) self.assertRunIsNotInDropdown(self.runs[0], data) @@ -677,5 +668,83 @@ class TestGraphView(unittest.TestCase): self.assertRunIsSelected(self.runs[3], data) +class TestGraphView(unittest.TestCase): + GRAPH_ENDPOINT = '/admin/airflow/graph?dag_id={dag_id}'.format( + dag_id=ViewWithDateTimeAndNumRunsAndDagRunsFormTester.DAG_ID + ) + + @classmethod + def setUpClass(cls): + super(TestGraphView, cls).setUpClass() + + def setUp(self): + super(TestGraphView, self).setUp() + self.tester = ViewWithDateTimeAndNumRunsAndDagRunsFormTester( + self, self.GRAPH_ENDPOINT) + self.tester.setUp() + + def tearDown(self): + self.tester.tearDown() + super(TestGraphView, self).tearDown() + + @classmethod + def tearDownClass(cls): + super(TestGraphView, cls).tearDownClass() + + def test_dt_nr_dr_form_default_parameters(self): + self.tester.test_with_default_parameters() + + def test_dt_nr_dr_form_with_execution_date_parameter_only(self): + self.tester.test_with_execution_date_parameter_only() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_parmeters_only(self): + self.tester.test_with_base_date_and_num_runs_parmeters_only() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_outside(self): + self.tester.test_with_base_date_and_num_runs_and_execution_date_outside() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_within(self): + self.tester.test_with_base_date_and_num_runs_and_execution_date_within() + + +class TestGanttView(unittest.TestCase): + GANTT_ENDPOINT = '/admin/airflow/gantt?dag_id={dag_id}'.format( + dag_id=ViewWithDateTimeAndNumRunsAndDagRunsFormTester.DAG_ID + ) + + @classmethod + def setUpClass(cls): + super(TestGanttView, cls).setUpClass() + + def setUp(self): + super(TestGanttView, self).setUp() + self.tester = ViewWithDateTimeAndNumRunsAndDagRunsFormTester( + self, self.GANTT_ENDPOINT) + self.tester.setUp() + + def tearDown(self): + self.tester.tearDown() + super(TestGanttView, self).tearDown() + + @classmethod + def tearDownClass(cls): + super(TestGanttView, cls).tearDownClass() + + def test_dt_nr_dr_form_default_parameters(self): + self.tester.test_with_default_parameters() + + def test_dt_nr_dr_form_with_execution_date_parameter_only(self): + self.tester.test_with_execution_date_parameter_only() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_parmeters_only(self): + self.tester.test_with_base_date_and_num_runs_parmeters_only() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_outside(self): + self.tester.test_with_base_date_and_num_runs_and_execution_date_outside() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_within(self): + self.tester.test_with_base_date_and_num_runs_and_execution_date_within() + + if __name__ == '__main__': unittest.main() http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ef7fe870/tests/www_rbac/test_views.py ---------------------------------------------------------------------- diff --git a/tests/www_rbac/test_views.py b/tests/www_rbac/test_views.py index cb81f62..71dcab0 100644 --- a/tests/www_rbac/test_views.py +++ b/tests/www_rbac/test_views.py @@ -516,25 +516,21 @@ class TestVersionView(TestBase): self.check_content_in_response('Version Info', resp) -class TestGraphView(TestBase): - DAG_ID = 'dag_for_testing_graph_view' +class ViewWithDateTimeAndNumRunsAndDagRunsFormTester: + DAG_ID = 'dag_for_testing_dt_nr_dr_form' DEFAULT_DATE = datetime(2017, 9, 1) RUNS_DATA = [ - ('dag_run_for_testing_graph_view_4', datetime(2018, 4, 4)), - ('dag_run_for_testing_graph_view_3', datetime(2018, 3, 3)), - ('dag_run_for_testing_graph_view_2', datetime(2018, 2, 2)), - ('dag_run_for_testing_graph_view_1', datetime(2018, 1, 1)), + ('dag_run_for_testing_dt_nr_dr_form_4', datetime(2018, 4, 4)), + ('dag_run_for_testing_dt_nr_dr_form_3', datetime(2018, 3, 3)), + ('dag_run_for_testing_dt_nr_dr_form_2', datetime(2018, 2, 2)), + ('dag_run_for_testing_dt_nr_dr_form_1', datetime(2018, 1, 1)), ] - GRAPH_ENDPOINT = '/graph?dag_id={dag_id}'.format( - dag_id=DAG_ID - ) - @classmethod - def setUpClass(cls): - super(TestGraphView, cls).setUpClass() + def __init__(self, test, endpoint): + self.test = test + self.endpoint = endpoint def setUp(self): - super(TestGraphView, self).setUp() from airflow.www_rbac.views import dagbag from airflow.utils.state import State dag = DAG(self.DAG_ID, start_date=self.DEFAULT_DATE) @@ -550,65 +546,60 @@ class TestGraphView(TestBase): self.runs.append(run) def tearDown(self): - self.session.query(DagRun).filter( + self.test.session.query(DagRun).filter( DagRun.dag_id == self.DAG_ID).delete() - self.session.commit() - self.session.close() - super(TestGraphView, self).tearDown() - - @classmethod - def tearDownClass(cls): - super(TestGraphView, cls).tearDownClass() + self.test.session.commit() + self.test.session.close() def assertBaseDateAndNumRuns(self, base_date, num_runs, data): - self.assertNotIn('name="base_date" value="{}"'.format(base_date), data) - self.assertNotIn('<option selected="" value="{}">{}</option>'.format( + self.test.assertNotIn('name="base_date" value="{}"'.format(base_date), data) + self.test.assertNotIn('<option selected="" value="{}">{}</option>'.format( num_runs, num_runs), data) def assertRunIsNotInDropdown(self, run, data): - self.assertNotIn(run.execution_date.isoformat(), data) - self.assertNotIn(run.run_id, data) + self.test.assertNotIn(run.execution_date.isoformat(), data) + self.test.assertNotIn(run.run_id, data) def assertRunIsInDropdownNotSelected(self, run, data): - self.assertIn('<option value="{}">{}</option>'.format( + self.test.assertIn('<option value="{}">{}</option>'.format( run.execution_date.isoformat(), run.run_id), data) def assertRunIsSelected(self, run, data): - self.assertIn('<option selected value="{}">{}</option>'.format( + self.test.assertIn('<option selected value="{}">{}</option>'.format( run.execution_date.isoformat(), run.run_id), data) - def test_graph_view_default_parameters(self): + def test_with_default_parameters(self): """ - Tests graph view with no URL parameter. + Tests view with no URL parameter. Should show all dag runs in the drop down. Should select the latest dag run. Should set base date to current date (not asserted) """ - response = self.client.get( - self.GRAPH_ENDPOINT + response = self.test.client.get( + self.endpoint ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') - self.assertIn('Base date:', data) - self.assertIn('Number of runs:', data) + self.test.assertIn('Base date:', data) + self.test.assertIn('Number of runs:', data) self.assertRunIsSelected(self.runs[0], data) self.assertRunIsInDropdownNotSelected(self.runs[1], data) self.assertRunIsInDropdownNotSelected(self.runs[2], data) self.assertRunIsInDropdownNotSelected(self.runs[3], data) - def test_graph_view_with_execution_date_parameter_only(self): + def test_with_execution_date_parameter_only(self): """ - Tests graph view with execution_date URL parameter. + Tests view with execution_date URL parameter. Scenario: click link from dag runs view. Should only show dag runs older than execution_date in the drop down. Should select the particular dag run. Should set base date to execution date. """ - response = self.client.get( - self.GRAPH_ENDPOINT + '&execution_date={}'.format( + response = self.test.client.get( + self.endpoint + '&execution_date={}'.format( self.runs[1].execution_date.isoformat()) ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') self.assertBaseDateAndNumRuns( self.runs[1].execution_date, @@ -619,19 +610,19 @@ class TestGraphView(TestBase): self.assertRunIsInDropdownNotSelected(self.runs[2], data) self.assertRunIsInDropdownNotSelected(self.runs[3], data) - def test_graph_view_with_base_date_and_num_runs_parmeters_only(self): + def test_with_base_date_and_num_runs_parmeters_only(self): """ - Tests graph view with base_date and num_runs URL parameters. + Tests view with base_date and num_runs URL parameters. Should only show dag runs older than base_date in the drop down, limited to num_runs. Should select the latest dag run. Should set base date and num runs to submitted values. """ - response = self.client.get( - self.GRAPH_ENDPOINT + '&base_date={}&num_runs=2'.format( + response = self.test.client.get( + self.endpoint + '&base_date={}&num_runs=2'.format( self.runs[1].execution_date.isoformat()) ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') self.assertBaseDateAndNumRuns(self.runs[1].execution_date, 2, data) self.assertRunIsNotInDropdown(self.runs[0], data) @@ -639,21 +630,21 @@ class TestGraphView(TestBase): self.assertRunIsInDropdownNotSelected(self.runs[2], data) self.assertRunIsNotInDropdown(self.runs[3], data) - def test_graph_view_with_base_date_and_num_runs_and_execution_date_outside(self): + def test_with_base_date_and_num_runs_and_execution_date_outside(self): """ - Tests graph view with base_date and num_runs and execution-date URL parameters. + Tests view with base_date and num_runs and execution-date URL parameters. Scenario: change the base date and num runs and press "Go", the selected execution date is outside the new range. Should only show dag runs older than base_date in the drop down. Should select the latest dag run within the range. Should set base date and num runs to submitted values. """ - response = self.client.get( - self.GRAPH_ENDPOINT + '&base_date={}&num_runs=42&execution_date={}'.format( + response = self.test.client.get( + self.endpoint + '&base_date={}&num_runs=42&execution_date={}'.format( self.runs[1].execution_date.isoformat(), self.runs[0].execution_date.isoformat()) ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') self.assertBaseDateAndNumRuns(self.runs[1].execution_date, 42, data) self.assertRunIsNotInDropdown(self.runs[0], data) @@ -661,21 +652,21 @@ class TestGraphView(TestBase): self.assertRunIsInDropdownNotSelected(self.runs[2], data) self.assertRunIsInDropdownNotSelected(self.runs[3], data) - def test_graph_view_with_base_date_and_num_runs_and_execution_date_within(self): + def test_with_base_date_and_num_runs_and_execution_date_within(self): """ - Tests graph view with base_date and num_runs and execution-date URL parameters. + Tests view with base_date and num_runs and execution-date URL parameters. Scenario: change the base date and num runs and press "Go", the selected execution date is within the new range. Should only show dag runs older than base_date in the drop down. Should select the dag run with the execution date. Should set base date and num runs to submitted values. """ - response = self.client.get( - self.GRAPH_ENDPOINT + '&base_date={}&num_runs=5&execution_date={}'.format( + response = self.test.client.get( + self.endpoint + '&base_date={}&num_runs=5&execution_date={}'.format( self.runs[2].execution_date.isoformat(), self.runs[3].execution_date.isoformat()) ) - self.assertEqual(response.status_code, 200) + self.test.assertEqual(response.status_code, 200) data = response.data.decode('utf-8') self.assertBaseDateAndNumRuns(self.runs[2].execution_date, 5, data) self.assertRunIsNotInDropdown(self.runs[0], data) @@ -684,5 +675,83 @@ class TestGraphView(TestBase): self.assertRunIsSelected(self.runs[3], data) +class TestGraphView(TestBase): + GRAPH_ENDPOINT = '/graph?dag_id={dag_id}'.format( + dag_id=ViewWithDateTimeAndNumRunsAndDagRunsFormTester.DAG_ID + ) + + @classmethod + def setUpClass(cls): + super(TestGraphView, cls).setUpClass() + + def setUp(self): + super(TestGraphView, self).setUp() + self.tester = ViewWithDateTimeAndNumRunsAndDagRunsFormTester( + self, self.GRAPH_ENDPOINT) + self.tester.setUp() + + def tearDown(self): + self.tester.tearDown() + super(TestGraphView, self).tearDown() + + @classmethod + def tearDownClass(cls): + super(TestGraphView, cls).tearDownClass() + + def test_dt_nr_dr_form_default_parameters(self): + self.tester.test_with_default_parameters() + + def test_dt_nr_dr_form_with_execution_date_parameter_only(self): + self.tester.test_with_execution_date_parameter_only() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_parmeters_only(self): + self.tester.test_with_base_date_and_num_runs_parmeters_only() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_outside(self): + self.tester.test_with_base_date_and_num_runs_and_execution_date_outside() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_within(self): + self.tester.test_with_base_date_and_num_runs_and_execution_date_within() + + +class TestGanttView(TestBase): + GANTT_ENDPOINT = '/gantt?dag_id={dag_id}'.format( + dag_id=ViewWithDateTimeAndNumRunsAndDagRunsFormTester.DAG_ID + ) + + @classmethod + def setUpClass(cls): + super(TestGanttView, cls).setUpClass() + + def setUp(self): + super(TestGanttView, self).setUp() + self.tester = ViewWithDateTimeAndNumRunsAndDagRunsFormTester( + self, self.GANTT_ENDPOINT) + self.tester.setUp() + + def tearDown(self): + self.tester.tearDown() + super(TestGanttView, self).tearDown() + + @classmethod + def tearDownClass(cls): + super(TestGanttView, cls).tearDownClass() + + def test_dt_nr_dr_form_default_parameters(self): + self.tester.test_with_default_parameters() + + def test_dt_nr_dr_form_with_execution_date_parameter_only(self): + self.tester.test_with_execution_date_parameter_only() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_parmeters_only(self): + self.tester.test_with_base_date_and_num_runs_parmeters_only() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_outside(self): + self.tester.test_with_base_date_and_num_runs_and_execution_date_outside() + + def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_within(self): + self.tester.test_with_base_date_and_num_runs_and_execution_date_within() + + if __name__ == '__main__': unittest.main()
