This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new 5cf454b feat: Scheduling queries from SQL Lab (#7416) (#7446)
5cf454b is described below
commit 5cf454b4647c6f52b0e7ae49ec5251a46703dd31
Author: Dave Smith <[email protected]>
AuthorDate: Fri May 3 16:45:19 2019 -0700
feat: Scheduling queries from SQL Lab (#7416) (#7446)
* Merge lastest from master into lyft-release-sp8 (#7405)
* filter out all nan series (#7313)
* improve not rich tooltip (#7345)
* Create issue_label_bot.yaml (#7341)
* fix: do not save colors without a color scheme (#7347)
* [wtforms] Strip leading/trailing whitespace (#7084)
* [schema] Updating the datasources schema (#5451)
* limit tables/views returned if schema is not provided (#7358)
* limit tables/views returned if schema is not provided
* fix typo
* improve code performance
* handle the case when table name or view name does not present a schema
* Add type anno (#7342)
* Updated local dev instructions to include missing step
* First pass at type annotations
* [schema] Updating the base column schema (#5452)
* Update 937d04c16b64_update_datasources.py (#7361)
* Feature flag for client cache (#7348)
* Feature flag for client cache
* Fix integration test
* Revert "Fix integration test"
This reverts commit 58434ab98a015d6e96db4a97f26255aa282d989d.
* Feature flag for client cache
* Fix integration tests
* Add feature flag to config.py
* Add another feature check
* Fix more integration tests
* Fix raw HTML in SliceAdder (#7338)
* remove backendSync.json (#7331)
* [bubbles] issue when using duplicated metrics (#7087)
* SUPERSET-7: Docker compose config version breaks on Ubuntu 16.04 (#7359)
* SUPERSET-8: Update text in docs copyright footer (#7360)
* SUPERSET-7: Docker compose config version breaks on Ubuntu 16.04
* SUPERSET-8: Extra text in docs copyright footer
* [schema] Adding commits and removing unnecessary foreign-key definitions
(#7371)
* Store last selected dashboard in sessionStorage (#7181)
* Store last selected dashboard in sessionStorage
* Fix tests
* [schema] Updating the base metric schema (#5453)
* Fix NoneType bug & fill the test recipients with original recipients if
empty (#7365)
* feat: see Presto row and array data types (#7391)
* feat: see Presto row and array data types
* fix: address PR comments
* fix: lint and build issues
* fix: add types
* Incorporate feedback from initial PR (prematurely merged to
lyft-release-sp8) (#7415)
* add stronger type hints where possible
* fix: lint issues and add select_star func in Hive
* add missing pkg init
* fix: build issues
* fix: pylint issues
* fix: use logging instead of print
* feat: view presto row objects in data grid
* fix: address feedback
* fix: spacing
* Workaround for no results returned (#7442)
* feat: view presto row objects in data grid (#7436)
* feat: view presto row objects in data grid
* fix: address feedback
* fix: spacing
* feat: Scheduling queries from SQL Lab (#7416)
* Lightweight pipelines POC
* Add docs
* Minor fixes
* Remove Lyft URL
* Use enum
* Minor fix
* Fix unit tests
* Mark props as required
---
docs/installation.rst | 78 +++++++++++++++
superset/assets/package-lock.json | 46 ++++++++-
superset/assets/package.json | 1 +
.../src/SqlLab/components/QueryAutoRefresh.jsx | 10 +-
.../src/SqlLab/components/ScheduleQueryButton.jsx | 109 +++++++++++++++++++++
.../assets/src/SqlLab/components/SqlEditor.jsx | 14 +++
superset/assets/src/featureFlags.ts | 3 +-
superset/views/sql_lab.py | 7 +-
8 files changed, 259 insertions(+), 9 deletions(-)
diff --git a/docs/installation.rst b/docs/installation.rst
index 1f406c9..b7c83e5 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -811,6 +811,84 @@ in this dictionary are made available for users to use in
their SQL.
'my_crazy_macro': lambda x: x*2,
}
+**Scheduling queries**
+
+You can optionally allow your users to schedule queries directly in SQL Lab.
+This is done by addding extra metadata to saved queries, which are then picked
+up by an external scheduled (like [Apache
Airflow](https://airflow.apache.org/)).
+
+To allow scheduled queries, add the following to your `config.py`:
+
+.. code-block:: python
+
+ FEATURE_FLAGS = {
+ # Configuration for scheduling queries from SQL Lab. This information
is
+ # collected when the user clicks "Schedule query", and saved into the
`extra`
+ # field of saved queries.
+ # See: https://github.com/mozilla-services/react-jsonschema-form
+ 'SCHEDULED_QUERIES': {
+ 'JSONSCHEMA': {
+ 'title': 'Schedule',
+ 'description': (
+ 'In order to schedule a query, you need to specify when it
'
+ 'should start running, when it should stop running, and
how '
+ 'often it should run. You can also optionally specify '
+ 'dependencies that should be met before the query is '
+ 'executed. Please read the documentation for best
practices '
+ 'and more information on how to specify dependencies.'
+ ),
+ 'type': 'object',
+ 'properties': {
+ 'output_table': {
+ 'type': 'string',
+ 'title': 'Output table name',
+ },
+ 'start_date': {
+ 'type': 'string',
+ 'format': 'date-time',
+ 'title': 'Start date',
+ },
+ 'end_date': {
+ 'type': 'string',
+ 'format': 'date-time',
+ 'title': 'End date',
+ },
+ 'schedule_interval': {
+ 'type': 'string',
+ 'title': 'Schedule interval',
+ },
+ 'dependencies': {
+ 'type': 'array',
+ 'title': 'Dependencies',
+ 'items': {
+ 'type': 'string',
+ },
+ },
+ },
+ },
+ 'UISCHEMA': {
+ 'schedule_interval': {
+ 'ui:placeholder': '@daily, @weekly, etc.',
+ },
+ 'dependencies': {
+ 'ui:help': (
+ 'Check the documentation for the correct format when '
+ 'defining dependencies.'
+ ),
+ },
+ },
+ },
+ }
+
+This feature flag is based on
[react-jsonschema-form](https://github.com/mozilla-services/react-jsonschema-form),
+and will add a button called "Schedule Query" to SQL Lab. When the button is
+clicked, a modal will show up where the user can add the metadata required for
+scheduling the query.
+
+This information can then be retrieved from the endpoint
`/savedqueryviewapi/api/read`
+and used to schedule the queries that have `scheduled_queries` in their JSON
+metadata. For schedulers other than Airflow, additional fields can be easily
+added to the configuration file above.
Celery Flower
-------------
diff --git a/superset/assets/package-lock.json
b/superset/assets/package-lock.json
index 704e6f3..e5075d9 100644
--- a/superset/assets/package-lock.json
+++ b/superset/assets/package-lock.json
@@ -5791,8 +5791,7 @@
"co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz";,
- "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
- "dev": true
+ "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
},
"coa": {
"version": "2.0.2",
@@ -6071,8 +6070,7 @@
"core-js": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.0.tgz";,
- "integrity":
"sha512-kLRC6ncVpuEW/1kwrOXYX6KQASCVtrh1gQr/UiaVgFlf9WE5Vp+lNe5+h3LuMr5PAucWnnEXwH0nQHRH/gpGtw==",
- "dev": true
+ "integrity":
"sha512-kLRC6ncVpuEW/1kwrOXYX6KQASCVtrh1gQr/UiaVgFlf9WE5Vp+lNe5+h3LuMr5PAucWnnEXwH0nQHRH/gpGtw=="
},
"core-util-is": {
"version": "1.0.2",
@@ -13233,6 +13231,11 @@
"integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=",
"dev": true
},
+ "lodash.topath": {
+ "version": "4.5.2",
+ "resolved":
"https://registry.npmjs.org/lodash.topath/-/lodash.topath-4.5.2.tgz";,
+ "integrity": "sha1-NhY1Hzu6YZlKCTGYlmC9AyVP0Ak="
+ },
"lodash.uniq": {
"version": "4.5.0",
"resolved":
"https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz";,
@@ -17458,6 +17461,41 @@
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.6.3.tgz";,
"integrity":
"sha512-u7FDWtthB4rWibG/+mFbVd5FvdI20yde86qKGx4lVUTWmPlSWQ4QxbBIrrs+HnXGbxOUlUzTAP/VDmvCwaP2yA=="
},
+ "react-jsonschema-form": {
+ "version": "1.2.0",
+ "resolved":
"https://registry.npmjs.org/react-jsonschema-form/-/react-jsonschema-form-1.2.0.tgz";,
+ "integrity":
"sha512-rR77qoFiQ5TxDYwsJz8UWmDner4jQ4xMnDqeV6Nvg7GtoEyOUoTVkI/SBMEzfXuF/piWZXYjquP96Hy/2L7C+Q==",
+ "requires": {
+ "ajv": "^5.2.3",
+ "babel-runtime": "^6.26.0",
+ "core-js": "^2.5.7",
+ "lodash.topath": "^4.5.2",
+ "prop-types": "^15.5.8"
+ },
+ "dependencies": {
+ "ajv": {
+ "version": "5.5.2",
+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz";,
+ "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
+ "requires": {
+ "co": "^4.6.0",
+ "fast-deep-equal": "^1.0.0",
+ "fast-json-stable-stringify": "^2.0.0",
+ "json-schema-traverse": "^0.3.0"
+ }
+ },
+ "fast-deep-equal": {
+ "version": "1.1.0",
+ "resolved":
"https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz";,
+ "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ="
+ },
+ "json-schema-traverse": {
+ "version": "0.3.1",
+ "resolved":
"https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz";,
+ "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A="
+ }
+ }
+ },
"react-lifecycles-compat": {
"version": "3.0.4",
"resolved":
"https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz";,
diff --git a/superset/assets/package.json b/superset/assets/package.json
index 6914c73..6edb971 100644
--- a/superset/assets/package.json
+++ b/superset/assets/package.json
@@ -117,6 +117,7 @@
"react-dom": "^16.4.1",
"react-gravatar": "^2.6.1",
"react-hot-loader": "^4.3.6",
+ "react-jsonschema-form": "^1.2.0",
"react-map-gl": "^4.0.10",
"react-markdown": "^3.3.0",
"react-redux": "^5.0.2",
diff --git a/superset/assets/src/SqlLab/components/QueryAutoRefresh.jsx
b/superset/assets/src/SqlLab/components/QueryAutoRefresh.jsx
index 6db56d7..8ad02ad 100644
--- a/superset/assets/src/SqlLab/components/QueryAutoRefresh.jsx
+++ b/superset/assets/src/SqlLab/components/QueryAutoRefresh.jsx
@@ -41,10 +41,18 @@ class QueryAutoRefresh extends React.PureComponent {
const { queries, queriesLastUpdate } = this.props;
const now = new Date().getTime();
+ // due to a race condition, queries can be marked as successful before the
+ // results key is set; this is a workaround until we fix the underlying
+ // problem
+ const isQueryRunning = q => (
+ ['running', 'started', 'pending', 'fetching'].indexOf(q.state) >= 0 ||
+ (q.state === 'success' && q.resultsKey === null)
+ );
+
return (
queriesLastUpdate > 0 &&
Object.values(queries).some(
- q => ['running', 'started', 'pending', 'fetching'].indexOf(q.state) >=
0 &&
+ q => isQueryRunning(q) &&
now - q.startDttm < MAX_QUERY_AGE_TO_POLL,
)
);
diff --git a/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx
b/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx
new file mode 100644
index 0000000..2e7e16e
--- /dev/null
+++ b/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import React from 'react';
+import PropTypes from 'prop-types';
+import Form from 'react-jsonschema-form';
+import { t } from '@superset-ui/translation';
+
+import Button from '../../components/Button';
+import ModalTrigger from '../../components/ModalTrigger';
+
+const propTypes = {
+ defaultLabel: PropTypes.string,
+ sql: PropTypes.string.isRequired,
+ schema: PropTypes.string.isRequired,
+ dbId: PropTypes.number.isRequired,
+ animation: PropTypes.bool,
+ onSchedule: PropTypes.func,
+};
+const defaultProps = {
+ defaultLabel: t('Undefined'),
+ animation: true,
+ onSchedule: () => {},
+};
+
+class ScheduleQueryButton extends React.PureComponent {
+ constructor(props) {
+ super(props);
+ this.state = {
+ description: '',
+ label: props.defaultLabel,
+ showSchedule: false,
+ };
+ this.toggleSchedule = this.toggleSchedule.bind(this);
+ this.onSchedule = this.onSchedule.bind(this);
+ this.onCancel = this.onCancel.bind(this);
+ this.onLabelChange = this.onLabelChange.bind(this);
+ this.onDescriptionChange = this.onDescriptionChange.bind(this);
+ }
+ onSchedule({ formData }) {
+ const query = {
+ label: this.state.label,
+ description: this.state.description,
+ db_id: this.props.dbId,
+ schema: this.props.schema,
+ sql: this.props.sql,
+ extra_json: JSON.stringify({ schedule_info: formData }),
+ };
+ this.props.onSchedule(query);
+ this.saveModal.close();
+ }
+ onCancel() {
+ this.saveModal.close();
+ }
+ onLabelChange(e) {
+ this.setState({ label: e.target.value });
+ }
+ onDescriptionChange(e) {
+ this.setState({ description: e.target.value });
+ }
+ toggleSchedule(e) {
+ this.setState({ target: e.target, showSchedule: !this.state.showSchedule
});
+ }
+ renderModalBody() {
+ return (
+ <Form
+ schema={window.featureFlags.SCHEDULED_QUERIES.JSONSCHEMA}
+ uiSchema={window.featureFlags.SCHEDULED_QUERIES.UISCHEMA}
+ onSubmit={this.onSchedule}
+ />
+ );
+ }
+ render() {
+ return (
+ <span className="ScheduleQueryButton">
+ <ModalTrigger
+ ref={(ref) => { this.saveModal = ref; }}
+ modalTitle={t('Schedule Query')}
+ modalBody={this.renderModalBody()}
+ triggerNode={
+ <Button bsSize="small" className="toggleSchedule"
onClick={this.toggleSchedule}>
+ <i className="fa fa-calendar" /> {t('Schedule Query')}
+ </Button>
+ }
+ bsSize="medium"
+ />
+ </span>
+ );
+ }
+}
+ScheduleQueryButton.propTypes = propTypes;
+ScheduleQueryButton.defaultProps = defaultProps;
+
+export default ScheduleQueryButton;
diff --git a/superset/assets/src/SqlLab/components/SqlEditor.jsx
b/superset/assets/src/SqlLab/components/SqlEditor.jsx
index 960a4af..f4495af 100644
--- a/superset/assets/src/SqlLab/components/SqlEditor.jsx
+++ b/superset/assets/src/SqlLab/components/SqlEditor.jsx
@@ -36,6 +36,7 @@ import LimitControl from './LimitControl';
import TemplateParamsEditor from './TemplateParamsEditor';
import SouthPane from './SouthPane';
import SaveQuery from './SaveQuery';
+import ScheduleQueryButton from './ScheduleQueryButton';
import ShareSqlLabQuery from './ShareSqlLabQuery';
import Timer from '../../components/Timer';
import Hotkeys from '../../components/Hotkeys';
@@ -43,6 +44,7 @@ import SqlEditorLeftBar from './SqlEditorLeftBar';
import AceEditorWrapper from './AceEditorWrapper';
import { STATE_BSSTYLE_MAP } from '../constants';
import RunQueryActionButton from './RunQueryActionButton';
+import { FeatureFlag, isFeatureEnabled } from '../../featureFlags';
const SQL_EDITOR_PADDING = 10;
const SQL_TOOLBAR_HEIGHT = 51;
@@ -313,6 +315,18 @@ class SqlEditor extends React.PureComponent {
sql={this.state.sql}
/>
</span>
+ {isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) &&
+ <span className="m-r-5">
+ <ScheduleQueryButton
+ defaultLabel={qe.title}
+ sql={qe.sql}
+ className="m-r-5"
+ onSchedule={this.props.actions.saveQuery}
+ schema={qe.schema}
+ dbId={qe.dbId}
+ />
+ </span>
+ }
<span className="m-r-5">
<SaveQuery
defaultLabel={qe.title}
diff --git a/superset/assets/src/featureFlags.ts
b/superset/assets/src/featureFlags.ts
index 8638a54..450ad2c 100644
--- a/superset/assets/src/featureFlags.ts
+++ b/superset/assets/src/featureFlags.ts
@@ -22,6 +22,7 @@ export enum FeatureFlag {
SCOPED_FILTER = 'SCOPED_FILTER',
OMNIBAR = 'OMNIBAR',
CLIENT_CACHE = 'CLIENT_CACHE',
+ SCHEDULED_QUERIES = 'SCHEDULED_QUERIES',
}
export type FeatureFlagMap = {
@@ -39,5 +40,5 @@ export function initFeatureFlags(featureFlags:
FeatureFlagMap) {
}
export function isFeatureEnabled(feature: FeatureFlag) {
- return !!window.featureFlags[feature];
+ return window && window.featureFlags && !!window.featureFlags[feature];
}
diff --git a/superset/views/sql_lab.py b/superset/views/sql_lab.py
index adbdd46..b9d1f2e 100644
--- a/superset/views/sql_lab.py
+++ b/superset/views/sql_lab.py
@@ -116,9 +116,10 @@ class SavedQueryView(SupersetModelView, DeleteMixin):
class SavedQueryViewApi(SavedQueryView):
list_columns = [
- 'label', 'sqlalchemy_uri', 'user_email', 'schema', 'description',
- 'sql']
- show_columns = ['label', 'db_id', 'schema', 'description', 'sql']
+ 'id', 'label', 'sqlalchemy_uri', 'user_email', 'schema', 'description',
+ 'sql', 'extra_json']
+ show_columns = [
+ 'label', 'db_id', 'schema', 'description', 'sql', 'extra_json']
add_columns = show_columns
edit_columns = add_columns
