Repository: incubator-airflow Updated Branches: refs/heads/master 7c0f8373f -> 004272b15
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/airflow/www/templates/airflow/chart.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/chart.html b/airflow/www/templates/airflow/chart.html index 1901af5..91b8be1 100644 --- a/airflow/www/templates/airflow/chart.html +++ b/airflow/www/templates/airflow/chart.html @@ -21,54 +21,21 @@ <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}"> </form> </div> -<div style="clear: both;"></div> +<div style="clear: both;">{{ chart |safe }}</div> <hr/> {% endblock %} {% block tail %} {{ super() }} - <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script> - <script src="{{ url_for('static', filename='highcharts.js') }}"></script> - <script src="/ck/static/chartkick.js"></script> <script> // We blur task_ids in demo mode $( document ).ready(function() { - if ("{{ demo_mode }}" == "True") { - d3.select("svg") - .append("filter") - .attr("id", "blur-effect-1") - .append("feGaussianBlur") - .attr("stdDeviation", 3); - d3.selectAll("g.highcharts-legend-item text").style("filter", "url(#blur-effect-1)"); - } - $('#uncheck').click(function(){ - chart = Highcharts.charts[0]; - $(chart.series).each(function(){ - this.setVisible(false, false); - }); - chart.redraw(); - }); - $('#check').click(function(){ - chart = Highcharts.charts[0]; - $(chart.series).each(function(){ - this.setVisible(true, false); - }); - chart.redraw(); - }); }); </script> - <div class="container"> - {% line_chart data with height=height library=chart_options%} - <div class="text-center"> - <button class="btn" id="uncheck">Hide all series</button> - <button class="btn" id="check">Show all series</button> - </div> - </div> + <div class="container"></div> - <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script> <script src="{{ admin_static.url( filename='vendor/bootstrap-daterangepicker/daterangepicker.js') }}"></script> - <script src="{{ admin_static.url(filename='admin/js/form-1.0.0.js') }}"></script> <script> {% endblock %} http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/airflow/www/templates/airflow/dag.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 22b312e..89ce2d6 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -4,8 +4,8 @@ {% block title %}Airflow - DAG {{ dag.dag_id }}{% endblock %} {% block head_css %} - {{ super() }} {{ lib.form_css() }} + {{ super() }} {% endblock %} {% block body %} @@ -186,8 +186,8 @@ </div> {% endblock %} {% block tail %} - {{ super() }} {{ lib.form_js() }} + {{ super() }} <script> function updateQueryStringParameter(uri, key, value) { var re = new RegExp("([?&])" + key + "=.*?(&|$)", "i"); http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/airflow/www/templates/airflow/gantt.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/gantt.html b/airflow/www/templates/airflow/gantt.html index ada2bc8..cd58cc9 100644 --- a/airflow/www/templates/airflow/gantt.html +++ b/airflow/www/templates/airflow/gantt.html @@ -2,66 +2,48 @@ {% block head_css %} {{ super() }} -<link href="{{ admin_static.url(filename='vendor/bootstrap-daterangepicker/daterangepicker-bs2.css') }}" rel="stylesheet"> +<link href="{{ admin_static.url(filename='vendor/bootstrap-daterangepicker/daterangepicker-bs2.css') }}" rel="stylesheet"/> +<link type="text/css" href="{{ url_for('static', filename='gantt.css') }}" rel="stylesheet" /> +<link type="text/css" href="{{ url_for('static', filename='tree.css') }}" rel="stylesheet" /> {% endblock %} {% block body %} {{ super() }} <form method="get"> - <div class="form-inline"> - Run:<input type="hidden" value="{{ dag.dag_id }}" name="dag_id"> - {{ form.execution_date(class_="form-control") | safe }} - <input type="submit" value="Go" class="btn btn-default" action="" method="get"> - <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}"> - </div> + <div class="form-inline"> + Run:<input type="hidden" value="{{ dag.dag_id }}" name="dag_id"> + {{ form.execution_date(class_="form-control") | safe }} + <input type="submit" value="Go" class="btn btn-default" action="" method="get"> + <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}"> + </div> </form> -<div id="container"></div> +<div style="clear: both;"></div> +<div class="container"> + <div class="gantt"></div> +</div> {% endblock %} {% block tail %} - {{ super() }} - <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script> - <script src="{{ url_for('static', filename='highcharts.js') }}"></script> - <script src="{{ url_for('static', filename='highcharts-more.js') }}"> - </script> - <script src="{{ admin_static.url( - filename='vendor/bootstrap-daterangepicker/daterangepicker.js') }}"></script> - <script src="{{ admin_static.url(filename='admin/js/form-1.0.0.js') }}"></script> - <script> - $( document ).ready(function() { - execution_date = '{{ execution_date }}'; - hc = {{ hc|safe }}; - hc.plotOptions.series.point = { - events: { - click: function(p){ - call_modal(this.category, execution_date); - } - } - }; - - hc.tooltip = { - formatter: function() { - duration = new Date(this.point.high - this.point.low); - return "From " + - Highcharts.dateFormat('%H:%M:%S', new Date(this.point.low)) + - " to " + - Highcharts.dateFormat('%H:%M:%S', new Date(this.point.high)) + - "<br>Duration: " + - Highcharts.dateFormat('%H:%M:%S', new Date(duration)); - } - } - $("#container").highcharts(hc); - - // We blur task_ids in demo mode - if ("{{ demo_mode }}" == "True") { - d3.select("svg") - .append("filter") - .attr("id", "blur-effect-1") - .append("feGaussianBlur") - .attr("stdDeviation", 3); - d3.selectAll("g.highcharts-xaxis-labels text").style("filter", "url(#blur-effect-1)"); - } - }); - </script> +{{ super() }} +<script src="{{ admin_static.url( + filename='vendor/bootstrap-daterangepicker/daterangepicker.js') }}"></script> +<script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script> +<script src="{{ url_for('static', filename='d3.tip.v0.6.3.js') }}"></script>/ +<script src="{{ url_for('static', filename='gantt-chart-d3v2.js') }}"></script> +<script> + $( document ).ready(function() { + var dag_id = '{{ dag.dag_id }}'; + var task_id = ''; + var exection_date = ''; + data = {{ data |safe }}; + var gantt = d3.gantt() + .taskTypes(data.taskNames) + .taskStatus(data.taskStatus) + .height(data.height) + .selector('.gantt') + .tickFormat("%H:%M:%S"); + gantt(data.tasks); + }); +</script> {% endblock %} http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/airflow/www/templates/airflow/highchart.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/highchart.html b/airflow/www/templates/airflow/highchart.html deleted file mode 100644 index 728caca..0000000 --- a/airflow/www/templates/airflow/highchart.html +++ /dev/null @@ -1,183 +0,0 @@ -{% extends "airflow/master.html" %} -{% block head_css %} -{{ super() }} -<link rel="stylesheet" type="text/css" - href="{{ url_for("static", filename="main.css") }}"> -<link rel="stylesheet" type="text/css" - href="{{ url_for("static", filename="dataTables.bootstrap.css") }}"> -<style> - pre { - margin: 0px; - padding: 0px; - border: none; - background: none; - background-color: #FFF; - } - .highcharts-tooltip>span { - background: rgba(255,255,255,0.85); - border: 1px solid silver; - border-radius: 3px; - box-shadow: 1px 1px 2px #888; - padding: 8px; - z-index: 2; - } -.panel-heading .accordion-toggle:after { - /* symbol for "opening" panels */ - font-family: 'Glyphicons Halflings'; /* essential for enabling glyphicon */ - content: "\e114"; /* adjust as needed, taken from bootstrap.css */ - float: right; /* adjust as needed */ - color: grey; /* adjust as needed */ -} -.panel-heading .accordion-toggle.collapsed:after { - /* symbol for "collapsed" panels */ - content: "\e080"; /* adjust as needed, taken from bootstrap.css */ -} -{% if embed %} -/* override padding formatting for flush margins */ -.container { - padding: 0px; -} -body { - padding: 0px; - background-color: #fff; -} -.navbar { - display: none; -} -{% endif %} -</style> -{% endblock %} -{% block title %} -{{ title }} -{% endblock %} -{% block body %} -{{ super() }} -{% if embed %} - <div id="chart_body"> - <img src="{{ url_for('static', filename='loading.gif') }}" width="50px"> - </div> -{% else %} - <div id="container"> - <h2> - <span id="label">{{ label }}</span> - <a href="/admin/chart/edit/?id={{ chart.id }}" > - <span class="glyphicon glyphicon-edit" aria-hidden="true" ></span> - </a> - </h2> - <div id="error" style="display: none;" class="alert alert-danger" role="alert"> - <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span> - <span id="error_msg">Oops.</span> - </div> - {% if chart.show_sql %} - <div class="panel panel-default"> - <div class="panel-heading" role="tab" id="headingTwo"> - <h4 class="panel-title"> - <a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#sql_panel" aria-expanded="true" aria-controls="sql_panel"> - SQL - </a> - </h4> - </div> - <div id="sql_panel" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingTwo"> - <div class="panel-body" id="sql_panel_body"> - {{ sql }} - </div> - </div> - </div> - {% endif %} - {% if chart.chart_type != "datatable" %} - <div id="chart_section" class="panel panel-default"> - <div class="panel-heading" role="tab" id="headingTwo"> - <h4 class="panel-title"> - <a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#chart_panel" aria-expanded="true" aria-controls="chart_panel"> - Chart - </a> - </h4> - </div> - <div id="chart_panel" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingTwo"> - <div class="panel-body"> - <div id="chart_body"> - <img src="{{ url_for('static', filename='loading.gif') }}" width="50px"> - </div> - </div> - </div> - </div> - {% endif %} - {% if chart.show_datatable or chart.chart_type == "datatable" %} - <div id="datatable_section" class="panel panel-default"> - <div class="panel-heading" role="tab" id="headingTwo"> - <h4 class="panel-title"> - <a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#datatable_panel" aria-expanded="true" aria-controls="datatable_panel"> - Data - </a> - </h4> - </div> - <div id="datatable_panel" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingTwo"> - <div class="panel-body" id="datatable_panel_body"> - <table id="datatable" class="dataframe table table-bordered table-striped no-wrap"></table> - <img id="loading" src="{{ url_for('static', filename='loading.gif') }}" width="50px"> - </div> - </div> - </div> - {% endif %} - </div> -{% endif %} -{% endblock %} -{% block tail %} - {{ super() }} - <script src="{{ url_for('static', filename='jquery.dataTables.min.js') }}"></script> - <script src="{{ url_for('static', filename='highcharts.js') }}"></script> - <script src="{{ url_for('static', filename='highcharts-more.js') }}"> - </script> - <script src="{{ url_for('static', filename='heatmap.js') }}"></script> - <script src="{{ url_for('static', filename='heatmap-canvas.js') }}"></script> - <script> - function error(msg){ - $('#error_msg').html(msg); - $('#error').show(); - $('#loading').hide(); - $('#chart_section').hide(1000); - $('#datatable_section').hide(1000); - } - function warn(msg){ - $('#error_msg').html(msg); - $('#error').show(); - } - $( document ).ready(function() { - Highcharts.setOptions({ - colors: [ - "#FF5A5F", "#007A87", "#7B0051", "#00D1C1", "#8CE071", "#FFB400", - "#FFAA91", "#B4A76C", "#9CA299", "#565A5C" - ], - }); - url = "{{ url_for('airflow.chart_data') }}" + location.search; - $.getJSON(url, function(payload) { - $('#loading').hide(); - $("#sql_panel_body").html(payload.sql_html); - $("#label").html(payload.label); - if (payload.state == "SUCCESS") { - {% if chart.chart_type != "datatable" %} - $('#chart_body').css('width', '100%'); - $('#chart_body').css('height', '{{ chart.height }}'); - $('#chart_body').highcharts(payload.hc); - {% endif %} - {% if chart.show_datatable or chart.chart_type == "datatable" %} - $('#datatable').dataTable( { - "data": payload.data.data, - "columns": payload.data.columns, - "scrollX": true, - "iDisplayLength": 100, - }); - {% endif %} - } - else { - error(payload.error); - } - if ('warning' in payload) - warn(payload.warning); - }).fail(function(jqxhr, textStatus, err) { - error( textStatus + ': ' + err ); - }); - }); - </script> - -{% endblock %} http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/airflow/www/templates/airflow/nvd3.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/nvd3.html b/airflow/www/templates/airflow/nvd3.html new file mode 100644 index 0000000..49c35ea --- /dev/null +++ b/airflow/www/templates/airflow/nvd3.html @@ -0,0 +1,175 @@ +{% extends "airflow/master.html" %} +{% block head_css %} +{{ super() }} +<link rel="stylesheet" type="text/css" + href="{{ url_for("static", filename="main.css") }}"> +<link rel="stylesheet" type="text/css" + href="{{ url_for("static", filename="dataTables.bootstrap.css") }}"> +<link rel="stylesheet" type="text/css" + href="{{ url_for("static", filename="nv.d3.css") }}"> +<style> + pre { + margin: 0px; + padding: 0px; + border: none; + background: none; + background-color: #FFF; + } +.panel-heading .accordion-toggle:after { + /* symbol for "opening" panels */ + font-family: 'Glyphicons Halflings'; /* essential for enabling glyphicon */ + content: "\e114"; /* adjust as needed, taken from bootstrap.css */ + float: right; /* adjust as needed */ + color: grey; /* adjust as needed */ +} +.panel-heading .accordion-toggle.collapsed:after { + /* symbol for "collapsed" panels */ + content: "\e080"; /* adjust as needed, taken from bootstrap.css */ +} +{% if embed %} +/* override padding formatting for flush margins */ +.container { + padding: 0px; +} +body { + padding: 0px; + background-color: #fff; +} +.navbar { + display: none; +} +{% endif %} +</style> +{% endblock %} +{% block title %} +{{ title }} +{% endblock %} +{% block body %} +{{ super() }} +{% if embed %} + <div id="chart_body"> + <img src="{{ url_for('static', filename='loading.gif') }}" width="50px"> + </div> +{% else %} + <div id="container"> + <h2> + <span id="label">{{ label }}</span> + <a href="/admin/chart/edit/?id={{ chart.id }}" > + <span class="glyphicon glyphicon-edit" aria-hidden="true" ></span> + </a> + </h2> + <div id="error" style="display: none;" class="alert alert-danger" role="alert"> + <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span> + <span id="error_msg">Oops.</span> + </div> + {% if chart.show_sql %} + <div class="panel panel-default"> + <div class="panel-heading" role="tab" id="headingTwo"> + <h4 class="panel-title"> + <a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#sql_panel" aria-expanded="true" aria-controls="sql_panel"> + SQL + </a> + </h4> + </div> + <div id="sql_panel" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingTwo"> + <div class="panel-body" id="sql_panel_body"> + {{ sql }} + </div> + </div> + </div> + {% endif %} + {% if chart.chart_type != "datatable" %} + <div id="chart_section" class="panel panel-default"> + <div class="panel-heading" role="tab" id="headingTwo"> + <h4 class="panel-title"> + <a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#chart_panel" aria-expanded="true" aria-controls="chart_panel"> + Chart + </a> + </h4> + </div> + <div id="chart_panel" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingTwo"> + <div class="panel-body"> + <div id="chart_body"> + <img src="{{ url_for('static', filename='loading.gif') }}" width="50px"> + </div> + </div> + </div> + </div> + {% endif %} + {% if chart.show_datatable or chart.chart_type == "datatable" %} + <div id="datatable_section" class="panel panel-default"> + <div class="panel-heading" role="tab" id="headingTwo"> + <h4 class="panel-title"> + <a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#datatable_panel" aria-expanded="true" aria-controls="datatable_panel"> + Data + </a> + </h4> + </div> + <div id="datatable_panel" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingTwo"> + <div class="panel-body" id="datatable_panel_body"> + <table id="datatable" class="dataframe table table-bordered table-striped no-wrap"></table> + <img id="loading" src="{{ url_for('static', filename='loading.gif') }}" width="50px"> + </div> + </div> + </div> + {% endif %} + </div> +{% endif %} +{% endblock %} +{% block tail %} + {{ super() }} + <script src="{{ url_for('static', filename='jquery.dataTables.min.js') }}"></script> + <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script> + <script src="{{ url_for('static', filename='nv.d3.js') }}"></script> + <script> + function error(msg){ + $('#error_msg').html(msg); + $('#error').show(); + $('#loading').hide(); + $('#chart_section').hide(1000); + $('#datatable_section').hide(1000); + } + function warn(msg){ + $('#error_msg').html(msg); + $('#error').show(); + } + $( document ).ready(function() { + colors: [ + "#FF5A5F", "#007A87", "#7B0051", "#00D1C1", "#8CE071", "#FFB400", + "#FFAA91", "#B4A76C", "#9CA299", "#565A5C" + ]; + url = "{{ url_for('airflow.chart_data') }}" + location.search; + $.getJSON(url, function(payload) { + $('#loading').hide(); + if (payload.error !== undefined) { + $('#chart_body').html('<div class="alert alert-danger">' + payload.error + '</div>'); + } + $("#sql_panel_body").html(payload.sql_html); + $("#label").html(payload.label); + if (payload.state == "SUCCESS") { + {% if chart.chart_type != "datatable" %} + $('#chart_body').css('width', '100%'); + console.log(payload); + $('#chart_body').html(payload.htmlcontent); + {% endif %} + {% if chart.show_datatable or chart.chart_type == "datatable" %} + $('#datatable').dataTable( { + "data": payload.data.data, + "columns": payload.data.columns, + "scrollX": true, + "iDisplayLength": 100, + }); + {% endif %} + } + else { + error(payload.error); + } + if ('warning' in payload) + warn(payload.warning); + }).fail(function(jqxhr, textStatus, err) { + error( textStatus + ': ' + err ); + }); + }); + </script> + +{% endblock %} http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/airflow/www/utils.py ---------------------------------------------------------------------- diff --git a/airflow/www/utils.py b/airflow/www/utils.py index d600cbb..7288a10 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -16,21 +16,23 @@ from future import standard_library standard_library.install_aliases() from builtins import str from builtins import object + from cgi import escape from io import BytesIO as IO import functools import gzip import dateutil.parser as dateparser import json +import time + from flask import after_this_request, request, Response from flask_login import current_user -from jinja2 import Template import wtforms from wtforms.compat import text_type from airflow import configuration, models, settings from airflow.utils.json import AirflowJsonEncoder -from airflow.utils.email import send_email + AUTHENTICATE = configuration.getboolean('webserver', 'AUTHENTICATE') @@ -87,6 +89,11 @@ def limit_sql(sql, limit, conn_type): return sql +def epoch(dttm): + """Returns an epoch-type date""" + return int(time.mktime(dttm.timetuple())) * 1000, + + def action_logging(f): ''' Decorator to log user actions http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/airflow/www/views.py ---------------------------------------------------------------------- diff --git a/airflow/www/views.py b/airflow/www/views.py index 6570adc..f26ca7b 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -12,31 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import sys import os import pkg_resources import socket import importlib - from functools import wraps from datetime import datetime, timedelta import dateutil.parser import copy from itertools import chain, product +import json from past.utils import old_div from past.builtins import basestring import inspect -import subprocess import traceback import sqlalchemy as sqla from sqlalchemy import or_, desc, and_ - -from flask import redirect, url_for, request, Markup, Response, current_app, render_template +from flask import ( + redirect, url_for, request, Markup, Response, current_app, render_template) from flask_admin import BaseView, expose, AdminIndexView from flask_admin.contrib.sqla import ModelView from flask_admin.actions import action @@ -45,7 +43,7 @@ from flask._compat import PY2 import jinja2 import markdown -import json +import nvd3 from wtforms import ( Form, SelectField, TextAreaField, PasswordField, StringField) @@ -291,6 +289,8 @@ class Airflow(BaseView): @wwwutils.gzipped # @cache.cached(timeout=3600, key_prefix=wwwutils.make_cache_key) def chart_data(self): + from airflow import macros + import pandas as pd session = settings.Session() chart_id = request.args.get('chart_id') csv = request.args.get('csv') == "true" @@ -301,6 +301,7 @@ class Airflow(BaseView): session.commit() session.close() + payload = {} payload['state'] = 'ERROR' payload['error'] = '' @@ -317,7 +318,6 @@ class Airflow(BaseView): "a Python dictionary. ") request_dict = {k: request.args.get(k) for k in request.args} - from airflow import macros args.update(request_dict) args['macros'] = macros sql = jinja2.Template(chart.sql).render(**args) @@ -329,11 +329,11 @@ class Airflow(BaseView): ) payload['label'] = label - import pandas as pd pd.set_option('display.max_colwidth', 100) hook = db.get_hook() try: - df = hook.get_pandas_df(wwwutils.limit_sql(sql, CHART_LIMIT, conn_type=db.conn_type)) + df = hook.get_pandas_df( + wwwutils.limit_sql(sql, CHART_LIMIT, conn_type=db.conn_type)) df = df.fillna(0) except Exception as e: payload['error'] += "SQL execution failed. Details: " + str(e) @@ -352,26 +352,25 @@ class Airflow(BaseView): if not payload['error'] and len(df) == 0: payload['error'] += "Empty result set. " elif ( - not payload['error'] and - chart.sql_layout == 'series' and - chart.chart_type != "datatable" and - len(df.columns) < 3): + not payload['error'] and + chart.sql_layout == 'series' and + chart.chart_type != "datatable" and + len(df.columns) < 3): payload['error'] += "SQL needs to return at least 3 columns. " elif ( - not payload['error'] and - chart.sql_layout == 'columns'and - len(df.columns) < 2): + not payload['error'] and + chart.sql_layout == 'columns'and + len(df.columns) < 2): payload['error'] += "SQL needs to return at least 2 columns. " elif not payload['error']: import numpy as np chart_type = chart.chart_type data = None - if chart_type == "datatable": - chart.show_datatable = True - if chart.show_datatable: + if chart.show_datatable or chart_type == "datatable": data = df.to_dict(orient="split") data['columns'] = [{'title': c} for c in data['columns']] + payload['data'] = data # Trying to convert time to something Highcharts likes x_col = 1 if chart.sql_layout == 'series' else 0 @@ -380,101 +379,14 @@ class Airflow(BaseView): # From string to datetime df[df.columns[x_col]] = pd.to_datetime( df[df.columns[x_col]]) + df[df.columns[x_col]] = df[df.columns[x_col]].apply( + lambda x: int(x.strftime("%s")) * 1000) except Exception as e: - raise AirflowException(str(e)) - df[df.columns[x_col]] = df[df.columns[x_col]].apply( - lambda x: int(x.strftime("%s")) * 1000) + payload['error'] = "Time conversion failed" - series = [] - colorAxis = None if chart_type == 'datatable': - payload['data'] = data payload['state'] = 'SUCCESS' return wwwutils.json_response(payload) - - elif chart_type == 'para': - df.rename(columns={ - df.columns[0]: 'name', - df.columns[1]: 'group', - }, inplace=True) - return Response( - response=df.to_csv(index=False), - status=200, - mimetype="application/text") - - elif chart_type == 'heatmap': - color_perc_lbound = float( - request.args.get('color_perc_lbound', 0)) - color_perc_rbound = float( - request.args.get('color_perc_rbound', 1)) - color_scheme = request.args.get('color_scheme', 'blue_red') - - if color_scheme == 'blue_red': - stops = [ - [color_perc_lbound, '#00D1C1'], - [ - color_perc_lbound + - ((color_perc_rbound - color_perc_lbound)/2), - '#FFFFCC' - ], - [color_perc_rbound, '#FF5A5F'] - ] - elif color_scheme == 'blue_scale': - stops = [ - [color_perc_lbound, '#FFFFFF'], - [color_perc_rbound, '#2222FF'] - ] - elif color_scheme == 'fire': - diff = float(color_perc_rbound - color_perc_lbound) - stops = [ - [color_perc_lbound, '#FFFFFF'], - [color_perc_lbound + 0.33*diff, '#FFFF00'], - [color_perc_lbound + 0.66*diff, '#FF0000'], - [color_perc_rbound, '#000000'] - ] - else: - stops = [ - [color_perc_lbound, '#FFFFFF'], - [ - color_perc_lbound + - ((color_perc_rbound - color_perc_lbound)/2), - '#888888' - ], - [color_perc_rbound, '#000000'], - ] - - xaxis_label = df.columns[1] - yaxis_label = df.columns[2] - data = [] - for row in df.itertuples(): - data.append({ - 'x': row[2], - 'y': row[3], - 'value': row[4], - }) - x_format = '{point.x:%Y-%m-%d}' \ - if chart.x_is_date else '{point.x}' - series.append({ - 'data': data, - 'borderWidth': 0, - 'colsize': 24 * 36e5, - 'turboThreshold': sys.float_info.max, - 'tooltip': { - 'headerFormat': '', - 'pointFormat': ( - df.columns[1] + ': ' + x_format + '<br/>' + - df.columns[2] + ': {point.y}<br/>' + - df.columns[3] + ': <b>{point.value}</b>' - ), - }, - }) - colorAxis = { - 'stops': stops, - 'minColor': '#FFFFFF', - 'maxColor': '#000000', - 'min': 50, - 'max': 2200, - } else: if chart.sql_layout == 'series': # User provides columns (series, x, y) @@ -495,63 +407,22 @@ class Airflow(BaseView): for col in df.columns: df[col] = df[col].astype(np.float) + df = df.fillna(0) + NVd3ChartClass = chart_mapping.get(chart.chart_type) + NVd3ChartClass = getattr(nvd3, NVd3ChartClass) + nvd3_chart = NVd3ChartClass(x_is_date=chart.x_is_date) + for col in df.columns: - series.append({ - 'name': col, - 'data': [ - (k, df[col][k]) - for k in df[col].keys() - if not np.isnan(df[col][k])] - }) - series = [serie for serie in sorted( - series, key=lambda s: s['data'][0][1], reverse=True)] - - if chart_type == "stacked_area": - stacking = "normal" - chart_type = 'area' - elif chart_type == "percent_area": - stacking = "percent" - chart_type = 'area' - else: - stacking = None - hc = { - 'chart': { - 'type': chart_type - }, - 'plotOptions': { - 'series': { - 'marker': { - 'enabled': False - } - }, - 'area': {'stacking': stacking}, - }, - 'title': {'text': ''}, - 'xAxis': { - 'title': {'text': xaxis_label}, - 'type': 'datetime' if chart.x_is_date else None, - }, - 'yAxis': { - 'title': {'text': yaxis_label}, - }, - 'colorAxis': colorAxis, - 'tooltip': { - 'useHTML': True, - 'backgroundColor': None, - 'borderWidth': 0, - }, - 'series': series, - } + nvd3_chart.add_serie(name=col, y=df[col].tolist(), x=df[col].index.tolist()) + try: + nvd3_chart.buildcontent() + payload['chart_type'] = nvd3_chart.__class__.__name__ + payload['htmlcontent'] = nvd3_chart.htmlcontent + except Exception as e: + payload['error'] = str(e) - if chart.y_log_scale: - hc['yAxis']['type'] = 'logarithmic' - hc['yAxis']['minorTickInterval'] = 0.1 - if 'min' in hc['yAxis']: - del hc['yAxis']['min'] payload['state'] = 'SUCCESS' - payload['hc'] = hc - payload['data'] = data payload['request_dict'] = request_dict return wwwutils.json_response(payload) @@ -565,8 +436,14 @@ class Airflow(BaseView): session.expunge_all() session.commit() session.close() - if chart.chart_type == 'para': - return self.render('airflow/para/para.html', chart=chart) + + NVd3ChartClass = chart_mapping.get(chart.chart_type) + if not NVd3ChartClass: + flash( + "Not supported anymore as the license was incompatible, " + "sorry", + "danger") + redirect('/admin/chart/') sql = "" if chart.show_sql: @@ -576,7 +453,7 @@ class Airflow(BaseView): HtmlFormatter(noclasses=True)) ) return self.render( - 'airflow/highchart.html', + 'airflow/nvd3.html', chart=chart, title="Airflow - Chart", sql=sql, @@ -1476,38 +1353,27 @@ class Airflow(BaseView): include_upstream=True, include_downstream=False) - all_data = [] max_duration = 0 + chart = nvd3.lineChart( + name="lineChart", x_is_date=True, height=600, width="1200") + for task in dag.tasks: - data = [] + y = [] + x = [] for ti in task.get_task_instances(session, start_date=min_date, end_date=base_date): if ti.duration: - data.append([ - ti.execution_date.isoformat(), - ti.duration - ]) if max_duration < ti.duration: max_duration = ti.duration - if data: - all_data.append({'data': data, 'name': task.task_id}) - - def divide_durations(all_data, denom): - for data in all_data: - for d in data['data']: - d[1] /= denom - - if 60*60 < max_duration: - unit = 'hours' - divide_durations(all_data, float(60*60)) - elif 60 < max_duration: - unit = 'minutes' - divide_durations(all_data, 60.0) - else: - unit = 'seconds' + + dttm = wwwutils.epoch(ti.execution_date) + x.append(dttm) + y.append(float(ti.duration) / (60*60)) + if x: + chart.add_serie(name=task.task_id, x=x, y=y) tis = dag.get_task_instances( - session, start_date=min_date, end_date=base_date) + session, start_date=min_date, end_date=base_date) dates = sorted(list({ti.execution_date for ti in tis})) max_date = max([ti.execution_date for ti in tis]) if dates else None @@ -1516,15 +1382,14 @@ class Airflow(BaseView): form = DateTimeWithNumRunsForm(data={'base_date': max_date, 'num_runs': num_runs}) + chart.buildhtml() return self.render( 'airflow/chart.html', dag=dag, - data=json.dumps(all_data), - chart_options={'yAxis': {'title': {'text': unit}}}, - height="700px", demo_mode=conf.getboolean('webserver', 'demo_mode'), root=root, form=form, + chart=chart, ) @expose('/landing_times') @@ -1553,18 +1418,23 @@ class Airflow(BaseView): include_upstream=True, include_downstream=False) - all_data = [] + chart = nvd3.lineChart( + name="lineChart", x_is_date=True, height=600, width="1200") for task in dag.tasks: - data = [] + y = [] + x = [] for ti in task.get_task_instances(session, start_date=min_date, end_date=base_date): + ts = ti.execution_date + if dag.schedule_interval: + ts = dag.following_schedule(ts) if ti.end_date: - ts = ti.execution_date - if dag.schedule_interval: - ts = dag.following_schedule(ts) + dttm = wwwutils.epoch(ti.execution_date) secs = old_div((ti.end_date - ts).total_seconds(), 60*60) - data.append([ti.execution_date.isoformat(), secs]) - all_data.append({'data': data, 'name': task.task_id}) + x.append(dttm) + y.append(secs) + if x: + chart.add_serie(name=task.task_id, x=x, y=y) tis = dag.get_task_instances( session, start_date=min_date, end_date=base_date) @@ -1579,9 +1449,8 @@ class Airflow(BaseView): return self.render( 'airflow/chart.html', dag=dag, - data=json.dumps(all_data), + chart=chart, height="700px", - chart_options={'yAxis': {'title': {'text': 'hours after 00:00'}}}, demo_mode=conf.getboolean('webserver', 'demo_mode'), root=root, form=form, @@ -1639,7 +1508,6 @@ class Airflow(BaseView): @login_required @wwwutils.action_logging def gantt(self): - session = settings.Session() dag_id = request.args.get('dag_id') dag = dagbag.get_dag(dag_id) @@ -1661,57 +1529,40 @@ class Airflow(BaseView): form = DateTimeForm(data={'execution_date': dttm}) tis = [ - ti - for ti in dag.get_task_instances(session, dttm, dttm) + ti for ti in dag.get_task_instances(session, dttm, dttm) if ti.start_date] tis = sorted(tis, key=lambda ti: ti.start_date) + tasks = [] - data = [] - for i, ti in enumerate(tis): - end_date = ti.end_date or datetime.now() - tasks += [ti.task_id] - color = State.color(ti.state) - data.append({ - 'x': i, - 'low': int(ti.start_date.strftime('%s')) * 1000, - 'high': int(end_date.strftime('%s')) * 1000, - 'color': color, + for ti in tis: + tasks.append({ + 'startDate': wwwutils.epoch(ti.start_date), + 'endDate': wwwutils.epoch(ti.end_date or datetime.now()), + 'isoStart': ti.start_date.isoformat()[:-4], + 'isoEnd': ti.end_date.isoformat()[:-4], + 'taskName': ti.task_id, + 'duration': "{}".format(ti.end_date - ti.start_date)[:-4], + 'status': ti.state, + 'executionDate': ti.execution_date.isoformat(), }) - height = (len(tis) * 25) + 50 + states = {ti.state:ti.state for ti in tis} + data = { + 'taskNames': [ti.task_id for ti in tis], + 'tasks': tasks, + 'taskStatus': states, + 'height': len(tis) * 25, + } + session.commit() session.close() - hc = { - 'chart': { - 'type': 'columnrange', - 'inverted': True, - 'height': height, - }, - 'xAxis': {'categories': tasks, 'alternateGridColor': '#FAFAFA'}, - 'yAxis': {'type': 'datetime'}, - 'title': { - 'text': None - }, - 'plotOptions': { - 'series': { - 'cursor': 'pointer', - 'minPointLength': 4, - }, - }, - 'legend': { - 'enabled': False - }, - 'series': [{ - 'data': data - }] - } return self.render( 'airflow/gantt.html', dag=dag, execution_date=dttm.isoformat(), form=form, - hc=json.dumps(hc, indent=4), - height=height, + data=json.dumps(data, indent=2), + base_date='', demo_mode=demo_mode, root=root, ) @@ -1984,12 +1835,10 @@ class ChartModelView(wwwutils.DataProfilingMixin, AirflowModelView): ('line', 'Line Chart'), ('spline', 'Spline Chart'), ('bar', 'Bar Chart'), - ('para', 'Parallel Coordinates'), ('column', 'Column Chart'), ('area', 'Overlapping Area Chart'), ('stacked_area', 'Stacked Area Chart'), ('percent_area', 'Percent Area Chart'), - ('heatmap', 'Heatmap'), ('datatable', 'No chart, data table only'), ], 'sql_layout': [ @@ -2014,6 +1863,18 @@ class ChartModelView(wwwutils.DataProfilingMixin, AirflowModelView): model.user_id = current_user.id model.last_modified = datetime.now() +chart_mapping = ( + ('line', 'lineChart'), + ('spline', 'lineChart'), + ('bar', 'multiBarChart'), + ('column', 'multiBarChart'), + ('area', 'stackedAreaChart'), + ('stacked_area', 'stackedAreaChart'), + ('percent_area', 'stackedAreaChart'), + ('datatable', 'datatable'), +) +chart_mapping = dict(chart_mapping) + class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView): verbose_name = "known event" @@ -2314,7 +2175,12 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView): @classmethod def alert_fernet_key(cls): - return conf.get('core', 'fernet_key') is None + fk = None + try: + fk = conf.get('core', 'fernet_key') + except: + pass + return fk is None @classmethod def is_secure(self): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 411a699..978c484 100644 --- a/setup.py +++ b/setup.py @@ -155,13 +155,13 @@ github_enterprise = ['Flask-OAuthlib>=0.9.1'] qds = ['qds-sdk>=1.9.0'] cloudant = ['cloudant>=0.5.9,<2.0'] # major update coming soon, clamp to 0.x - all_dbs = postgres + mysql + hive + mssql + hdfs + vertica + cloudant devel = ['lxml>=3.3.4', 'nose', 'nose-parameterized', 'mock', 'click', 'jira'] devel_minreq = devel + mysql + doc + password + s3 devel_hadoop = devel_minreq + hive + hdfs + webhdfs + kerberos devel_all = devel + all_dbs + doc + samba + s3 + slack + crypto + oracle + docker + def do_setup(): write_version() setup( @@ -177,34 +177,35 @@ def do_setup(): install_requires=[ 'alembic>=0.8.3, <0.9', 'babel>=1.3, <2.0', - 'chartkick>=0.4.2, < 0.5', 'croniter>=0.3.8, <0.4', 'dill>=0.2.2, <0.3', - 'python-daemon>=2.1.1, <2.2', 'flask>=0.10.1, <0.11', 'flask-admin==1.4.1', 'flask-cache>=0.13.1, <0.14', 'flask-login==0.2.11', - 'future>=0.15.0, <0.16', + 'flask-wtf==0.12', 'funcsigs>=0.4, <1', + 'future>=0.15.0, <0.16', 'gitpython>=2.0.2', 'gunicorn>=19.3.0, <19.4.0', # 19.4.? seemed to have issues 'jinja2>=2.7.3, <3.0', 'markdown>=2.5.2, <3.0', 'pandas>=0.15.2, <1.0.0', 'pygments>=2.0.1, <3.0', + 'python-daemon>=2.1.1, <2.2', 'python-dateutil>=2.3, <3', + 'python-nvd3==0.14.2', 'requests>=2.5.1, <3', 'setproctitle>=1.1.8, <2', 'sqlalchemy>=0.9.8', 'thrift>=0.9.2, <0.10', - 'Flask-WTF==0.12' ], extras_require={ 'all': devel_all, 'all_dbs': all_dbs, 'async': async, 'celery': celery, + 'cloudant': cloudant, 'crypto': crypto, 'devel': devel_minreq, 'devel_hadoop': devel_hadoop, @@ -212,26 +213,25 @@ def do_setup(): 'docker': docker, 'druid': druid, 'gcp_api': gcp_api, + 'github_enterprise': github_enterprise, 'hdfs': hdfs, 'hive': hive, 'jdbc': jdbc, + 'kerberos': kerberos, + 'ldap': ldap, 'mssql': mssql, 'mysql': mysql, 'oracle': oracle, + 'password': password, 'postgres': postgres, + 'qds': qds, 'rabbitmq': rabbitmq, 's3': s3, 'samba': samba, 'slack': slack, 'statsd': statsd, 'vertica': vertica, - 'ldap': ldap, 'webhdfs': webhdfs, - 'kerberos': kerberos, - 'password': password, - 'github_enterprise': github_enterprise, - 'qds': qds, - 'cloudant': cloudant }, classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -257,3 +257,4 @@ def do_setup(): if __name__ == "__main__": do_setup() + http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0a460081/tests/core.py ---------------------------------------------------------------------- diff --git a/tests/core.py b/tests/core.py index 5e6a4fd..8dfd81a 100644 --- a/tests/core.py +++ b/tests/core.py @@ -785,32 +785,13 @@ class WebUiTests(unittest.TestCase): response = self.app.get( '/admin/airflow/tree?num_runs=25&dag_id=example_bash_operator') assert "runme_0" in response.data.decode('utf-8') - # new Chartkick.LineChart(document.getElementById("chart-0"), [{"data": [["2015-11-17T16:53:08.652950", 9.866944444444444e-06]], "name": "run_after_loop"}, {"data": [["2015-11-17T16:53:08.652950", 0.0002858047222222222], ["2015-11-17T16:56:09.698921", 0.00028737944444444445]], "name": "runme_0"}, {"data": [["2015-11-17T16:53:08.652950", 0.0002863941666666666], ["2015-11-17T16:56:09.698921", 0.00029015249999999996]], "name": "runme_1"}, {"data": [["2015-11-17T16:53:08.652950", 0.0002860847222222222], ["2015-11-17T16:56:09.698921", 0.00029001583333333335]], "name": "runme_2"}, {"data": [["2015-11-17T16:53:08.652950", 8.166944444444444e-06], ["2015-11-17T16:56:09.698921", 1.2806944444444445e-05]], "name": "also_run_this"}], {"library": {"yAxis": {"title": {"text": "hours"}}}, "height": "700px"}); - - chartkick_regexp = 'new Chartkick.LineChart\(document.getElementById\("chart-\d+"\),(.+)\)\;' response = self.app.get( '/admin/airflow/duration?days=30&dag_id=example_bash_operator') assert "example_bash_operator" in response.data.decode('utf-8') - - chartkick_matched = re.search(chartkick_regexp, - response.data.decode('utf-8')) - assert chartkick_matched is not None, "chartkick_matched was none. Expected regex is: %s\nResponse was: %s" % ( - chartkick_regexp, - response.data.decode('utf-8')) - - # test that parameters to LineChart are well-formed json - try: - json.loads('[%s]' % chartkick_matched.group(1)) - except e: - assert False, "Exception while json parsing LineChart parameters: %s" % e - response = self.app.get( '/admin/airflow/landing_times?' 'days=30&dag_id=example_bash_operator') assert "example_bash_operator" in response.data.decode('utf-8') - chartkick_matched = re.search(chartkick_regexp, - response.data.decode('utf-8')) - assert chartkick_matched is not None response = self.app.get( '/admin/airflow/gantt?dag_id=example_bash_operator') assert "example_bash_operator" in response.data.decode('utf-8')
