This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new cecbba3 Summary: Introduce configs for default relative end time and
the epoch-tz (#6721)
cecbba3 is described below
commit cecbba3fe63cfb6ea294cec90f57e5282b6dfb6c
Author: agrawaldevesh <[email protected]>
AuthorDate: Fri Feb 1 16:34:13 2019 -0800
Summary: Introduce configs for default relative end time and the epoch-tz
(#6721)
Introduce a config DEFAULT_RELATIVE_END_TIME which is used when
computing the "Last X days". The default behavior (as currently) is to
let that be 'today', which actually means the 0th second of today. We
can also let it be 'now' which means the data as of now (ie at query
time).
Secondly, also introduce another config IS_EPOCH_S_TRULY_UTC, which
means that the logged time in epoch_s format is actually in UTC.
Currently (as the default) is that it it is in the local (superset
webserver) timezone.
** There is no backward incompatibility with thes config features since
the default behavior hasn't changed. **
---
superset/config.py | 10 ++++++++++
superset/connectors/sqla/models.py | 16 +++++++++++-----
superset/db_engine_specs.py | 2 +-
superset/utils/core.py | 25 +++++++++++++------------
superset/viz.py | 15 +++++++++------
5 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/superset/config.py b/superset/config.py
index 93720c3..8480ef8 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -566,6 +566,16 @@ WEBDRIVER_BASEURL = 'http://0.0.0.0:8080/'
# Send user to a link where they can report bugs
BUG_REPORT_URL = None
+# What is the Last N days relative in the time selector to:
+# 'today' means it is midnight (00:00:00) of today in the local timezone
+# 'now' means it is relative to the query issue time
+DEFAULT_RELATIVE_END_TIME = 'today'
+
+# Is epoch_s/epoch_ms datetime format supposed to be considered since UTC ?
+# If not, it is sassumed then the epoch_s/epoch_ms is seconds since 1/1/1970
+# localtime (in the tz where the superset webserver is running)
+IS_EPOCH_S_TRULY_UTC = False
+
try:
if CONFIG_PATH_ENV_VAR in os.environ:
diff --git a/superset/connectors/sqla/models.py
b/superset/connectors/sqla/models.py
index cfbea85..41b5dc7 100644
--- a/superset/connectors/sqla/models.py
+++ b/superset/connectors/sqla/models.py
@@ -127,12 +127,13 @@ class TableColumn(Model, BaseColumn):
return self.table
def get_time_filter(self, start_dttm, end_dttm):
+ is_epoch_in_utc = config.get('IS_EPOCH_S_TRULY_UTC', False)
col = self.get_sqla_col(label='__time')
l = [] # noqa: E741
if start_dttm:
- l.append(col >= text(self.dttm_sql_literal(start_dttm)))
+ l.append(col >= text(self.dttm_sql_literal(start_dttm,
is_epoch_in_utc)))
if end_dttm:
- l.append(col <= text(self.dttm_sql_literal(end_dttm)))
+ l.append(col <= text(self.dttm_sql_literal(end_dttm,
is_epoch_in_utc)))
return and_(*l)
def get_timestamp_expression(self, time_grain):
@@ -166,7 +167,7 @@ class TableColumn(Model, BaseColumn):
TableColumn.column_name == lookup_column.column_name).first()
return import_datasource.import_simple_obj(db.session, i_column,
lookup_obj)
- def dttm_sql_literal(self, dttm):
+ def dttm_sql_literal(self, dttm, is_epoch_in_utc):
"""Convert datetime object to a SQL expression string
If database_expression is empty, the internal dttm
@@ -179,10 +180,15 @@ class TableColumn(Model, BaseColumn):
if self.database_expression:
return self.database_expression.format(dttm.strftime('%Y-%m-%d
%H:%M:%S'))
elif tf:
+ if is_epoch_in_utc:
+ seconds_since_epoch = dttm.timestamp()
+ else:
+ seconds_since_epoch = (dttm - datetime(1970, 1,
1)).total_seconds()
+ seconds_since_epoch = int(seconds_since_epoch)
if tf == 'epoch_s':
- return str((dttm - datetime(1970, 1, 1)).total_seconds())
+ return str(seconds_since_epoch)
elif tf == 'epoch_ms':
- return str((dttm - datetime(1970, 1, 1)).total_seconds() *
1000.0)
+ return str(seconds_since_epoch * 1000)
return "'{}'".format(dttm.strftime(tf))
else:
s = self.table.database.db_engine_spec.convert_dttm(
diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py
index 937537b..3ae433c 100644
--- a/superset/db_engine_specs.py
+++ b/superset/db_engine_specs.py
@@ -138,7 +138,7 @@ class BaseEngineSpec(object):
@classmethod
def epoch_ms_to_dttm(cls):
- return cls.epoch_to_dttm().replace('{col}', '({col}/1000.000)')
+ return cls.epoch_to_dttm().replace('{col}', '({col}/1000)')
@classmethod
def get_datatype(cls, type_code):
diff --git a/superset/utils/core.py b/superset/utils/core.py
index 3e38ea5..efeff7e 100644
--- a/superset/utils/core.py
+++ b/superset/utils/core.py
@@ -920,7 +920,8 @@ def ensure_path_exists(path):
def get_since_until(time_range: Optional[str] = None,
since: Optional[str] = None,
until: Optional[str] = None,
- time_shift: Optional[str] = None) -> (datetime, datetime):
+ time_shift: Optional[str] = None,
+ relative_end: Optional[str] = None) -> (datetime,
datetime):
"""Return `since` and `until` date time tuple from string representations
of
time_range, since, until and time_shift.
@@ -946,13 +947,13 @@ def get_since_until(time_range: Optional[str] = None,
"""
separator = ' : '
- today = parse_human_datetime('today')
+ relative_end = parse_human_datetime(relative_end if relative_end else
'today')
common_time_frames = {
- 'Last day': (today - relativedelta(days=1), today),
- 'Last week': (today - relativedelta(weeks=1), today),
- 'Last month': (today - relativedelta(months=1), today),
- 'Last quarter': (today - relativedelta(months=3), today),
- 'Last year': (today - relativedelta(years=1), today),
+ 'Last day': (relative_end - relativedelta(days=1), relative_end),
+ 'Last week': (relative_end - relativedelta(weeks=1), relative_end),
+ 'Last month': (relative_end - relativedelta(months=1), relative_end),
+ 'Last quarter': (relative_end - relativedelta(months=3), relative_end),
+ 'Last year': (relative_end - relativedelta(years=1), relative_end),
}
if time_range:
@@ -969,17 +970,17 @@ def get_since_until(time_range: Optional[str] = None,
else:
rel, num, grain = time_range.split()
if rel == 'Last':
- since = today - relativedelta(**{grain: int(num)})
- until = today
+ since = relative_end - relativedelta(**{grain: int(num)})
+ until = relative_end
else: # rel == 'Next'
- since = today
- until = today + relativedelta(**{grain: int(num)})
+ since = relative_end
+ until = relative_end + relativedelta(**{grain: int(num)})
else:
since = since or ''
if since:
since = add_ago_to_since(since)
since = parse_human_datetime(since)
- until = parse_human_datetime(until or 'now')
+ until = parse_human_datetime(until) if until else relative_end
if time_shift:
time_shift = parse_human_timedelta(time_shift)
diff --git a/superset/viz.py b/superset/viz.py
index dc84630..51fab12 100644
--- a/superset/viz.py
+++ b/superset/viz.py
@@ -59,6 +59,7 @@ from superset.utils.core import (
config = app.config
stats_logger = config.get('STATS_LOGGER')
+relative_end = config.get('DEFAULT_RELATIVE_END_TIME', 'today')
METRIC_KEYS = [
'metric', 'metrics', 'percent_metrics', 'metric_2', 'secondary_metric',
@@ -280,9 +281,10 @@ class BaseViz(object):
# default order direction
order_desc = form_data.get('order_desc', True)
- since, until = utils.get_since_until(form_data.get('time_range'),
- form_data.get('since'),
- form_data.get('until'))
+ since, until = utils.get_since_until(relative_end=relative_end,
+
time_range=form_data.get('time_range'),
+ since=form_data.get('since'),
+ until=form_data.get('until'))
time_shift = form_data.get('time_shift', '')
self.time_shift = utils.parse_human_timedelta(time_shift)
from_dttm = None if since is None else (since - self.time_shift)
@@ -795,9 +797,10 @@ class CalHeatmapViz(BaseViz):
values[str(v / 10**9)] = obj.get(metric)
data[metric] = values
- start, end = utils.get_since_until(form_data.get('time_range'),
- form_data.get('since'),
- form_data.get('until'))
+ start, end = utils.get_since_until(relative_end=relative_end,
+
time_range=form_data.get('time_range'),
+ since=form_data.get('since'),
+ until=form_data.get('until'))
if not start or not end:
raise Exception('Please provide both time bounds (Since and
Until)')
domain = form_data.get('domain_granularity')
