[
https://issues.apache.org/jira/browse/AIRFLOW-2799?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16679841#comment-16679841
]
ASF GitHub Bot commented on AIRFLOW-2799:
-----------------------------------------
Fokko closed pull request #4061: [AIRFLOW-2799] Fix filtering UI objects by
datetime
URL: https://github.com/apache/incubator-airflow/pull/4061
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py
index 6d49fbcbb3..5adaa2f5c4 100644
--- a/airflow/utils/timezone.py
+++ b/airflow/utils/timezone.py
@@ -164,9 +164,9 @@ def datetime(*args, **kwargs):
return dt.datetime(*args, **kwargs)
-def parse(string):
+def parse(string, timezone=None):
"""
Parse a time string and return an aware datetime
:param string: time string
"""
- return pendulum.parse(string, tz=TIMEZONE)
+ return pendulum.parse(string, tz=timezone or TIMEZONE)
diff --git a/airflow/www/utils.py b/airflow/www/utils.py
index 6404d27f95..757d2268bf 100644
--- a/airflow/www/utils.py
+++ b/airflow/www/utils.py
@@ -38,7 +38,7 @@
from flask import after_this_request, request, Response
from flask_admin.model import filters
-from flask_admin.contrib.sqla.filters import FilterConverter
+import flask_admin.contrib.sqla.filters as sqlafilters
from flask_login import current_user
from airflow import configuration, models, settings
@@ -448,7 +448,43 @@ def __call__(self, field, **kwargs):
return wtforms.widgets.core.HTMLString(html)
-class UtcFilterConverter(FilterConverter):
+class UtcDateTimeFilterMixin(object):
+ def clean(self, value):
+ dt = super(UtcDateTimeFilterMixin, self).clean(value)
+ return timezone.make_aware(dt, timezone=timezone.utc)
+
+
+class UtcDateTimeEqualFilter(UtcDateTimeFilterMixin,
sqlafilters.DateTimeEqualFilter):
+ pass
+
+
+class UtcDateTimeNotEqualFilter(UtcDateTimeFilterMixin,
sqlafilters.DateTimeNotEqualFilter):
+ pass
+
+
+class UtcDateTimeGreaterFilter(UtcDateTimeFilterMixin,
sqlafilters.DateTimeGreaterFilter):
+ pass
+
+
+class UtcDateTimeSmallerFilter(UtcDateTimeFilterMixin,
sqlafilters.DateTimeSmallerFilter):
+ pass
+
+
+class UtcDateTimeBetweenFilter(UtcDateTimeFilterMixin,
sqlafilters.DateTimeBetweenFilter):
+ pass
+
+
+class UtcDateTimeNotBetweenFilter(UtcDateTimeFilterMixin,
sqlafilters.DateTimeNotBetweenFilter):
+ pass
+
+
+class UtcFilterConverter(sqlafilters.FilterConverter):
+
+ utcdatetime_filters = (UtcDateTimeEqualFilter, UtcDateTimeNotEqualFilter,
+ UtcDateTimeGreaterFilter, UtcDateTimeSmallerFilter,
+ UtcDateTimeBetweenFilter,
UtcDateTimeNotBetweenFilter,
+ sqlafilters.FilterEmpty)
+
@filters.convert('utcdatetime')
def conv_utcdatetime(self, column, name, **kwargs):
- return self.conv_datetime(column, name, **kwargs)
+ return [f(column, name, **kwargs) for f in self.utcdatetime_filters]
diff --git a/airflow/www_rbac/utils.py b/airflow/www_rbac/utils.py
index 0176a5312c..b25e1541ab 100644
--- a/airflow/www_rbac/utils.py
+++ b/airflow/www_rbac/utils.py
@@ -37,7 +37,10 @@
from pygments import highlight, lexers
from pygments.formatters import HtmlFormatter
from flask import request, Response, Markup, url_for
-from airflow import configuration
+from flask_appbuilder.models.sqla.interface import SQLAInterface
+import flask_appbuilder.models.sqla.filters as fab_sqlafilters
+import sqlalchemy as sqla
+from airflow import configuration, settings
from airflow.models import BaseOperator
from airflow.operators.subdag_operator import SubDagOperator
from airflow.utils import timezone
@@ -378,3 +381,69 @@ def get_chart_height(dag):
charts, that is charts that take up space based on the size of the
components within.
"""
return 600 + len(dag.tasks) * 10
+
+
+class UtcAwareFilterMixin(object):
+ def apply(self, query, value):
+ value = timezone.parse(value, timezone=timezone.utc)
+
+ return super(UtcAwareFilterMixin, self).apply(query, value)
+
+
+class UtcAwareFilterEqual(UtcAwareFilterMixin, fab_sqlafilters.FilterEqual):
+ pass
+
+
+class UtcAwareFilterGreater(UtcAwareFilterMixin,
fab_sqlafilters.FilterGreater):
+ pass
+
+
+class UtcAwareFilterSmaller(UtcAwareFilterMixin,
fab_sqlafilters.FilterSmaller):
+ pass
+
+
+class UtcAwareFilterNotEqual(UtcAwareFilterMixin,
fab_sqlafilters.FilterNotEqual):
+ pass
+
+
+class UtcAwareFilterConverter(fab_sqlafilters.SQLAFilterConverter):
+
+ conversion_table = (
+ (('is_utcdatetime', [UtcAwareFilterEqual,
+ UtcAwareFilterGreater,
+ UtcAwareFilterSmaller,
+ UtcAwareFilterNotEqual]),) +
+ fab_sqlafilters.SQLAFilterConverter.conversion_table
+ )
+
+
+class CustomSQLAInterface(SQLAInterface):
+ """
+ FAB does not know how to handle columns with leading underscores because
+ they are not supported by WTForm. This hack will remove the leading
+ '_' from the key to lookup the column names.
+
+ """
+ def __init__(self, obj):
+ super(CustomSQLAInterface, self).__init__(obj)
+
+ self.session = settings.Session()
+
+ def clean_column_names():
+ if self.list_properties:
+ self.list_properties = dict(
+ (k.lstrip('_'), v) for k, v in
self.list_properties.items())
+ if self.list_columns:
+ self.list_columns = dict(
+ (k.lstrip('_'), v) for k, v in self.list_columns.items())
+
+ clean_column_names()
+
+ def is_utcdatetime(self, col_name):
+ from airflow.utils.sqlalchemy import UtcDateTime
+ obj = self.list_columns[col_name].type
+ return isinstance(obj, UtcDateTime) or \
+ isinstance(obj, sqla.types.TypeDecorator) and \
+ isinstance(obj.impl, UtcDateTime)
+
+ filter_converter_class = UtcAwareFilterConverter
diff --git a/airflow/www_rbac/views.py b/airflow/www_rbac/views.py
index 8d0a8b0ea8..29e8da1b9c 100644
--- a/airflow/www_rbac/views.py
+++ b/airflow/www_rbac/views.py
@@ -41,7 +41,6 @@
from flask_appbuilder import BaseView, ModelView, expose, has_access
from flask_appbuilder.actions import action
from flask_appbuilder.models.sqla.filters import BaseFilter
-from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_babel import lazy_gettext
from past.builtins import unicode
from pygments import highlight, lexers
@@ -1839,27 +1838,7 @@ class AirflowModelView(ModelView):
list_widget = AirflowModelListWidget
page_size = PAGE_SIZE
- class CustomSQLAInterface(SQLAInterface):
- """
- FAB does not know how to handle columns with leading underscores
because
- they are not supported by WTForm. This hack will remove the leading
- '_' from the key to lookup the column names.
-
- """
- def __init__(self, obj):
- super(AirflowModelView.CustomSQLAInterface, self).__init__(obj)
-
- self.session = settings.Session()
-
- def clean_column_names():
- if self.list_properties:
- self.list_properties = dict(
- (k.lstrip('_'), v) for k, v in
self.list_properties.items())
- if self.list_columns:
- self.list_columns = dict(
- (k.lstrip('_'), v) for k, v in
self.list_columns.items())
-
- clean_column_names()
+ CustomSQLAInterface = wwwutils.CustomSQLAInterface
class SlaMissModelView(AirflowModelView):
diff --git a/tests/www/test_views.py b/tests/www/test_views.py
index 734b28bcdc..87b5ff720b 100644
--- a/tests/www/test_views.py
+++ b/tests/www/test_views.py
@@ -770,5 +770,23 @@ def
test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_within(sel
self.tester.test_with_base_date_and_num_runs_and_execution_date_within()
+class TestTaskInstanceView(unittest.TestCase):
+ TI_ENDPOINT = '/admin/taskinstance/?flt2_execution_date_greater_than={}'
+
+ def setUp(self):
+ super(TestTaskInstanceView, self).setUp()
+ configuration.load_test_config()
+ app = application.create_app(testing=True)
+ app.config['WTF_CSRF_METHODS'] = []
+ self.app = app.test_client()
+
+ def test_start_date_filter(self):
+ resp = self.app.get(self.TI_ENDPOINT.format('2018-10-09+22:44:31'))
+ # We aren't checking the logic of the date filter itself (that is built
+ # in to flask-admin) but simply that our UTC conversion was run - i.e.
it
+ # doesn't blow up!
+ self.assertEqual(resp.status_code, 200)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/www_rbac/test_views.py b/tests/www_rbac/test_views.py
index 4b6d9d7d12..ef5899dc27 100644
--- a/tests/www_rbac/test_views.py
+++ b/tests/www_rbac/test_views.py
@@ -1375,5 +1375,18 @@ def test_log_success_for_user(self):
self.check_content_in_response('"metadata":', resp)
+class TestTaskInstanceView(TestBase):
+ TI_ENDPOINT = '/taskinstance/list/?_flt_0_execution_date={}'
+
+ def test_start_date_filter(self):
+ resp = self.client.get(self.TI_ENDPOINT.format(
+ self.percent_encode('2018-10-09 22:44:31')))
+ # We aren't checking the logic of the date filter itself (that is built
+ # in to FAB) but simply that our UTC conversion was run - i.e. it
+ # doesn't blow up!
+ self.check_content_in_response('List Task Instance', resp)
+ pass
+
+
if __name__ == '__main__':
unittest.main()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Filtering UI objects by datetime is broken
> -------------------------------------------
>
> Key: AIRFLOW-2799
> URL: https://issues.apache.org/jira/browse/AIRFLOW-2799
> Project: Apache Airflow
> Issue Type: Bug
> Components: ui, webserver
> Affects Versions: 1.10.0
> Environment: Debian Stretch, Python 3.5.3
> Reporter: Kevin Campbell
> Assignee: Ash Berlin-Taylor
> Priority: Major
> Fix For: 1.10.1
>
>
> On master (49fd23a3ee0269e2b974648f4a823c1d0b6c12ec) searching objects via
> the user interface is broken for datetime fields.
> Create a new installation
> Create a test dag (example_bash_operator)
> Start webserver and scheduler
> Enable dag
> On web UI, go to Browse > Task Instances
> Search for task instances with execution_date greater than 5 days ago
> You will get an exception
> {code:java}
> ____/ ( ( ) ) \___
> /( ( ( ) _ )) ) )\
> (( ( )( ) ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) (( ( ) .((_ ) . )_
> ( ( ) ( ( ) ) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ( (_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _ ) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )( ( _ ) _) _(_ ( (_ )
> (_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
> ((__) \\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> ( ( ( ( | | ) ) )\ )
> ( /(| / ( )) ) ) )) )
> ( ( ((((_(|)_))))) )
> ( ||\(|(|)|/|| )
> ( |(||(||)|||| )
> ( //|/l|||)|\\ \ )
> (/ / // /|//||||\\ \ \ \ _)
> -------------------------------------------------------------------------------
> Node: wave.diffractive.io
> -------------------------------------------------------------------------------
> Traceback (most recent call last):
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
> line 1116, in _execute_context
> context = constructor(dialect, self, conn, *args)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/default.py",
> line 649, in _init_compiled
> for key in compiled_params
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/default.py",
> line 649, in <genexpr>
> for key in compiled_params
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/sql/type_api.py",
> line 1078, in process
> return process_param(value, dialect)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy_utc/sqltypes.py",
> line 30, in process_bind_param
> raise ValueError('naive datetime is disallowed')
> ValueError: naive datetime is disallowed
> The above exception was the direct cause of the following exception:
> Traceback (most recent call last):
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask/app.py",
> line 1982, in wsgi_app
> response = self.full_dispatch_request()
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask/app.py",
> line 1614, in full_dispatch_request
> rv = self.handle_user_exception(e)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask/app.py",
> line 1517, in handle_user_exception
> reraise(exc_type, exc_value, tb)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask/_compat.py",
> line 33, in reraise
> raise value
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask/app.py",
> line 1612, in full_dispatch_request
> rv = self.dispatch_request()
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask/app.py",
> line 1598, in dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask_admin/base.py",
> line 69, in inner
> return self._run_view(f, *args, **kwargs)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask_admin/base.py",
> line 368, in _run_view
> return fn(self, *args, **kwargs)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask_admin/model/base.py",
> line 1818, in index_view
> view_args.search, view_args.filters)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/flask_admin/contrib/sqla/view.py",
> line 969, in get_list
> count = count_query.scalar() if count_query else None
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/orm/query.py",
> line 2843, in scalar
> ret = self.one()
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/orm/query.py",
> line 2814, in one
> ret = self.one_or_none()
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/orm/query.py",
> line 2784, in one_or_none
> ret = list(self)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/orm/query.py",
> line 2855, in __iter__
> return self._execute_and_instances(context)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/orm/query.py",
> line 2878, in _execute_and_instances
> result = conn.execute(querycontext.statement, self._params)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
> line 945, in execute
> return meth(self, multiparams, params)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/sql/elements.py",
> line 263, in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
> line 1053, in _execute_clauseelement
> compiled_sql, distilled_params
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
> line 1121, in _execute_context
> None, None)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
> line 1402, in _handle_dbapi_exception
> exc_info
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/util/compat.py",
> line 203, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/util/compat.py",
> line 186, in reraise
> raise value.with_traceback(tb)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
> line 1116, in _execute_context
> context = constructor(dialect, self, conn, *args)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/default.py",
> line 649, in _init_compiled
> for key in compiled_params
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/engine/default.py",
> line 649, in <genexpr>
> for key in compiled_params
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy/sql/type_api.py",
> line 1078, in process
> return process_param(value, dialect)
> File
> "/home/kev/.virtualenvs/airflow/local/lib/python3.5/site-packages/sqlalchemy_utc/sqltypes.py",
> line 30, in process_bind_param
> raise ValueError('naive datetime is disallowed')
> sqlalchemy.exc.StatementError: (builtins.ValueError) naive datetime is
> disallowed [SQL: 'SELECT count(%(count_2)s) AS count_1 \nFROM task_instance
> \nWHERE task_instance.execution_date > %(execution_date_1)s'] [parameters:
> [{}]]
> {code}
> This appears to have been introduced in
> https://issues.apache.org/jira/browse/AIRFLOW-288
> I've written a patch for this, which appears to resolve the issue. Will raise
> a PR.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
