This is an automated email from the ASF dual-hosted git repository. lyndsi pushed a commit to branch lyndsi/sql-lab-new-explore-button-functionality-and-move-save-dataset-to-split-save-button in repository https://gitbox.apache.org/repos/asf/superset.git
commit fe143804a6cfc3ae912a3148cc147457fbaf95d7 Author: Hugh A. Miles II <[email protected]> AuthorDate: Mon Jun 6 16:19:20 2022 +0000 Working POC > columns are loading into page --- .../superset-ui-core/src/query/DatasourceKey.ts | 3 +- .../superset-ui-core/src/query/types/Datasource.ts | 4 +- superset/common/query_context_factory.py | 2 + superset/common/query_context_processor.py | 5 +- superset/models/helpers.py | 50 ++++---- superset/models/sql_lab.py | 60 ++-------- superset/utils/core.py | 35 +++--- superset/views/core.py | 127 ++++++++++++++++++++- 8 files changed, 182 insertions(+), 104 deletions(-) diff --git a/superset-frontend/packages/superset-ui-core/src/query/DatasourceKey.ts b/superset-frontend/packages/superset-ui-core/src/query/DatasourceKey.ts index 38a38e10b1..6f40abb9e3 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/DatasourceKey.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/DatasourceKey.ts @@ -27,7 +27,8 @@ export default class DatasourceKey { constructor(key: string) { const [idStr, typeStr] = key.split('__'); this.id = parseInt(idStr, 10); - this.type = DatasourceType.Table; // default to SqlaTable model + this.type = + typeStr === 'table' ? DatasourceType.Table : DatasourceType.Druid; this.type = typeStr === 'query' ? DatasourceType.Query : this.type; } diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts b/superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts index 9639a000d0..389d2dce44 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts @@ -21,10 +21,8 @@ import { Metric } from './Metric'; export enum DatasourceType { Table = 'table', + Druid = 'druid', Query = 'query', - Dataset = 'dataset', - SlTable = 'sl_table', - SavedQuery = 'saved_query', } /** diff --git a/superset/common/query_context_factory.py b/superset/common/query_context_factory.py index dc43d28de9..c5a5d7cd1b 100644 --- a/superset/common/query_context_factory.py +++ b/superset/common/query_context_factory.py @@ -82,6 +82,8 @@ class QueryContextFactory: # pylint: disable=too-few-public-methods # pylint: disable=no-self-use def _convert_to_model(self, datasource: DatasourceDict) -> BaseDatasource: + from superset.dao.datasource.dao import DatasourceDAO + from superset.utils.core import DatasourceType return DatasourceDAO.get_datasource( session=db.session, diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py index 3b174dc7a2..49977aaa55 100644 --- a/superset/common/query_context_processor.py +++ b/superset/common/query_context_processor.py @@ -116,7 +116,7 @@ class QueryContextProcessor: and col != DTTM_ALIAS ) ] - + breakpoint() if invalid_columns: raise QueryObjectValidationError( _( @@ -124,7 +124,7 @@ class QueryContextProcessor: invalid_columns=invalid_columns, ) ) - + query_result = self.get_query_result(query_obj) annotation_data = self.get_annotation_data(query_obj) cache.set_query_result( @@ -185,7 +185,6 @@ class QueryContextProcessor: # support multiple queries from different data sources. # The datasource here can be different backend but the interface is common - # pylint: disable=import-outside-toplevel from superset.models.sql_lab import Query query = "" diff --git a/superset/models/helpers.py b/superset/models/helpers.py index 08efb59b60..f7f9aeff33 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -677,11 +677,7 @@ class ExploreMixin: @property def column_names(self): - return ["ethnic_minority", "gender"] - - @property - def columns(self): - return ["<col_name>"] + return [col.get('column_name') for col in self.columns] @property def offset(self): @@ -767,6 +763,10 @@ class ExploreMixin: # todo(hugh): apply filters for extended query query_str_ext = self.get_query_str_extended(qry) sql = query_str_ext.sql + + print('*****' * 5) + + # sql = "select count(*) from flights" status = QueryStatus.SUCCESS errors = None error_message = None @@ -1117,12 +1117,11 @@ class ExploreMixin: if granularity not in self.dttm_cols and granularity is not None: granularity = self.main_dttm_col - # columns_by_name: Dict[str, sa.Table] = { - # col.column_name: col for col in self.columns - # } # todo(hugh): fix this - columns_by_name = {} - + columns_by_name = { + col.get('column_name'): col for col in self.columns + } + # todo(hugh): how are we handling metrics # metrics_by_name: Dict[str, Column] = { # todo column vs metric? # m.metric_name: m for m in self.metrics @@ -1218,26 +1217,27 @@ class ExploreMixin: # template_processor=template_processor, ) # if groupby field equals a selected column - elif selected in columns_by_name: - outer = columns_by_name[selected].get_sqla_col() - else: - outer = literal_column(f"({selected})") - outer = self.make_sqla_column_compatible(outer, selected) + # elif selected in columns_by_name: + # outer = columns_by_name[selected].get_sqla_col() + # else: + # outer = literal_column(f"({selected})") + # outer = self.make_sqla_column_compatible(outer, selected) else: outer = self.adhoc_column_to_sqla( col=selected, # template_processor=template_processor ) - groupby_all_columns[outer.name] = outer - if not series_column_names or outer.name in series_column_names: - groupby_series_columns[outer.name] = outer - select_exprs.append(outer) + # groupby_all_columns[outer.name] = outer + # if not series_column_names or outer.name in series_column_names: + # groupby_series_columns[outer.name] = outer + # select_exprs.append(outer) elif columns: for selected in columns: - select_exprs.append( - columns_by_name[selected].get_sqla_col() - if selected in columns_by_name - else self.make_sqla_column_compatible(literal_column(selected)) - ) + # select_exprs.append( + # columns_by_name[selected].get_sqla_col() + # if selected in columns_by_name + # else self.make_sqla_column_compatible(literal_column(selected)) + # ) + select_exprs.append(selected) metrics_exprs = [] # todo(hugh): fix this @@ -1288,7 +1288,7 @@ class ExploreMixin: if not db_engine_spec.allows_hidden_ordeby_agg: select_exprs = utils.remove_duplicates(select_exprs + orderby_exprs) - qry = sa.select(select_exprs) + qry = sa.select([sa.column("YEAR")]) # todo(hugh) fix templating # tbl, cte = self.get_from_clause(template_processor) diff --git a/superset/models/sql_lab.py b/superset/models/sql_lab.py index 22f565032c..b6e318c10e 100644 --- a/superset/models/sql_lab.py +++ b/superset/models/sql_lab.py @@ -17,7 +17,7 @@ """A collection of ORM sqlalchemy models for SQL Lab""" import re from datetime import datetime -from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING +from typing import Any, Dict, List, Type import simplejson as json import sqlalchemy as sqla @@ -52,11 +52,9 @@ from superset.sqllab.limiting_factor import LimitingFactor from superset.superset_typing import ResultSetColumnType from superset.utils.core import GenericDataType, QueryStatus, user_label -if TYPE_CHECKING: - from superset.db_engine_specs import BaseEngineSpec - +from superset.superset_typing import ResultSetColumnType -class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-method +class Query(Model, ExtraJSONMixin, ExploreMixin): """ORM model for SQL query Now that SQL Lab support multi-statement execution, an entry in this @@ -174,6 +172,8 @@ class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-me @property def columns(self) -> List[ResultSetColumnType]: + # todo(hughhh): move this logic into a base class + from superset.utils.core import GenericDataType bool_types = ("BOOL",) num_types = ( "DOUBLE", @@ -189,11 +189,10 @@ class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-me ) date_types = ("DATE", "TIME") str_types = ("VARCHAR", "STRING", "CHAR") - columns = [] - col_type = "" + columns = [] for col in self.extra.get("columns", []): computed_column = {**col} - col_type = col.get("type") + col_type = col.get('type') if col_type and any(map(lambda t: t in col_type.upper(), str_types)): computed_column["type_generic"] = GenericDataType.STRING @@ -204,7 +203,7 @@ class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-me if col_type and any(map(lambda t: t in col_type.upper(), date_types)): computed_column["type_generic"] = GenericDataType.TEMPORAL - computed_column["column_name"] = col.get("name") + computed_column["column_name"] = col.get('name') computed_column["groupby"] = True columns.append(computed_column) return columns # type: ignore @@ -236,49 +235,6 @@ class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-me def db_engine_spec(self) -> Type["BaseEngineSpec"]: return self.database.db_engine_spec - @property - def owners_data(self) -> List[Dict[str, Any]]: - return [] - - @property - def uid(self) -> str: - return f"{self.id}__{self.type}" - - @property - def is_rls_supported(self) -> bool: - return False - - @property - def cache_timeout(self) -> int: - return 0 - - @property - def column_names(self) -> List[Any]: - return [col.get("column_name") for col in self.columns] - - @property - def offset(self) -> int: - return 0 - - @property - def main_dttm_col(self) -> Optional[str]: - for col in self.columns: - if col.get("is_dttm"): - return col.get("column_name") # type: ignore - return None - - @property - def dttm_cols(self) -> List[Any]: - return [col.get("column_name") for col in self.columns if col.get("is_dttm")] - - @property - def default_endpoint(self) -> str: - return "" - - @staticmethod - def get_extra_cache_keys(query_obj: Dict[str, Any]) -> List[str]: - return [] - class SavedQuery(Model, AuditMixinNullable, ExtraJSONMixin, ImportExportMixin): """ORM model for SQL query""" diff --git a/superset/utils/core.py b/superset/utils/core.py index 44d76ea533..ceb49268b1 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -1674,23 +1674,26 @@ def extract_dataframe_dtypes( "date": GenericDataType.TEMPORAL, } - columns_by_name: Dict[str, Any] = {} - if datasource: - for column in datasource.columns: - if isinstance(column, dict): - columns_by_name[column.get("column_name")] = column - else: - columns_by_name[column.column_name] = column - + # todo(hughhhh): can we make the column_object a Union + if datasource and datasource.type == "query": + columns_by_name = {column.get('column_name'): column for column in datasource.columns} + else: + columns_by_name = ( + {column.column_name: column for column in datasource.columns} + if datasource + else {} + ) + generic_types: List[GenericDataType] = [] for column in df.columns: column_object = columns_by_name.get(column) series = df[column] inferred_type = infer_dtype(series) - if isinstance(column_object, dict): # type: ignore + # todo(hughhhh): can we make the column_object a Union + if datasource.type == "query": generic_type = ( GenericDataType.TEMPORAL - if column_object and column_object.get("is_dttm") + if column_object and column_object.get('is_dttm') else inferred_type_map.get(inferred_type, GenericDataType.STRING) ) else: @@ -1733,15 +1736,9 @@ def get_time_filter_status( # pylint: disable=too-many-branches applied_time_extras: Dict[str, str], ) -> Tuple[List[Dict[str, str]], List[Dict[str, str]]]: - temporal_columns: Set[Any] - if datasource.type == "query": - temporal_columns = { - col.get("column_name") for col in datasource.columns if col.get("is_dttm") - } - else: - temporal_columns = { - col.column_name for col in datasource.columns if col.is_dttm - } + # todo(hugh): fix this + # temporal_columns = {col.column_name for col in datasource.columns if col.is_dttm} + temporal_columns = {} applied: List[Dict[str, str]] = [] rejected: List[Dict[str, str]] = [] time_column = applied_time_extras.get(ExtraFiltersTimeColumnType.TIME_COL) diff --git a/superset/views/core.py b/superset/views/core.py index 5236ebc494..75848a29c9 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -753,7 +753,6 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods self.__class__.__name__, ) initial_form_data = {} - form_data_key = request.args.get("form_data_key") if key is not None: command = GetExplorePermalinkCommand(key) @@ -777,6 +776,132 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods value = GetFormDataCommand(parameters).run() initial_form_data = json.loads(value) if value else {} + from superset.dao.datasource.dao import DatasourceDAO + from superset.utils.core import DatasourceType + from superset.models.helpers import ExploreMixin + + # Handle SIP-68 Models or explore view + # API will always use /explore/<datasource_type>/<int:datasource_id>/ to query + # new models to power any viz in explore + if datasource_id and datasource_type: + # 1. Query datasource object by type and id + datasource = DatasourceDAO.get_datasource( + session=db.session, + datasource_type=DatasourceType(datasource_type), + datasource_id=datasource_id, + ) + + # 2. Verify that it's an ExploreMixin + if isinstance(datasource, ExploreMixin): + # Handle Query object bootstrap + datasource_name = datasource.name if datasource else _("[Missing Dataset]") + form_data, slc = get_form_data( + use_slice_data=True, initial_form_data=initial_form_data + ) + + query_context = request.form.get("query_context") + + viz_type = form_data.get("viz_type", "table") + if not viz_type and datasource and datasource.default_endpoint: + return redirect(datasource.default_endpoint) + + # slc perms + slice_add_perm = security_manager.can_access("can_write", "Chart") + slice_overwrite_perm = is_owner(slc, g.user) if slc else False + slice_download_perm = security_manager.can_access("can_csv", "Superset") + + form_data["datasource"] = ( + str(datasource_id) + "__" + cast(str, datasource_type) + ) + + # On explore, merge legacy and extra filters into the form data + utils.convert_legacy_filters_into_adhoc(form_data) + utils.merge_extra_filters(form_data) + + # merge request url params + if request.method == "GET": + utils.merge_request_params(form_data, request.args) + + # handle save or overwrite + action = request.args.get("action") + + if action == "overwrite" and not slice_overwrite_perm: + return json_error_response( + _("You don't have the rights to ") + _("alter this ") + _("chart"), + status=403, + ) + + if action == "saveas" and not slice_add_perm: + return json_error_response( + _("You don't have the rights to ") + _("create a ") + _("chart"), + status=403, + ) + + if action in ("saveas", "overwrite") and datasource: + return self.save_or_overwrite_slice( + slc, + slice_add_perm, + slice_overwrite_perm, + slice_download_perm, + datasource.id, + datasource.type, + datasource.name, + query_context, + ) + standalone_mode = ReservedUrlParameters.is_standalone_mode() + force = request.args.get("force") in {"force", "1", "true"} + dummy_datasource_data: Dict[str, Any] = { + "type": datasource_type, + "name": datasource_name, + "columns": [], + "metrics": [], + "database": {"id": 0, "backend": ""}, + } + try: + datasource_data = ( + datasource.data if datasource else dummy_datasource_data + ) + except (SupersetException, SQLAlchemyError): + datasource_data = dummy_datasource_data + + columns: List[Dict[str, Any]] = [] + if datasource: + datasource_data["owners"] = datasource.owners_data + if isinstance(datasource, Query): + # todo(hughhh): set is_dttm + name -> column_name + # datasource_data["data"] = datasource.data + # move all this logic into the class for the property data + datasource_data["columns"] = datasource.columns + datasource_data["metrics"] = datasource.extra.get("metrics", []) + datasource_data["id"] = datasource_id + datasource_data["type"] = datasource_type + + bootstrap_data = { + "can_add": slice_add_perm, + "can_download": slice_download_perm, + "datasource": sanitize_datasource_data(datasource_data), + "form_data": form_data, + "datasource_id": datasource_id, + "datasource_type": datasource_type, + "slice": slc.data if slc else None, + "standalone": standalone_mode, + "force": force, + "user": bootstrap_user_data(g.user, include_perms=True), + "forced_height": request.args.get("height"), + "common": common_bootstrap_payload(), + } + + title = _("Explore - %(name)s", name=datasource.name) + return self.render_template( + "superset/basic.html", + bootstrap_data=json.dumps( + bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser + ), + entry="explore", + title=title.__str__(), + standalone_mode=standalone_mode, + ) + if not initial_form_data: slice_id = request.args.get("slice_id") dataset_id = request.args.get("dataset_id")
