villebro commented on a change in pull request #11499:
URL:
https://github.com/apache/incubator-superset/pull/11499#discussion_r538167530
##########
File path: superset/common/query_context.py
##########
@@ -75,6 +75,13 @@ def __init__( # pylint: disable=too-many-arguments
self.custom_cache_timeout = custom_cache_timeout
self.result_type = result_type or utils.ChartDataResultType.FULL
self.result_format = result_format or utils.ChartDataResultFormat.JSON
+ self.cache_values = {
+ "datasource": datasource,
+ "queries": queries,
+ "force": force,
+ "result_type": result_type,
+ "result_format": result_format,
Review comment:
I get this now, sorry for the confusion. It might make sense to add a
comment that `cache_values` is meant to track the original args as closely as
possible, not the properties of the resulting object.
##########
File path: superset/config.py
##########
@@ -406,6 +407,9 @@ def _try_json_readsha( # pylint: disable=unused-argument
# Cache for datasource metadata and query results
DATA_CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "null"}
+# store cache keys by datasource UID (via CacheKey) for custom
processing/invalidation
+STORE_CACHE_KEYS_IN_METADATA_DB = False
Review comment:
As this is currently enabled by default, we probably need to add a note
in `UPDATING.md` that this is now disabled by default.
##########
File path: superset/utils/async_query_manager.py
##########
@@ -0,0 +1,180 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import logging
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+import jwt
+import redis
+from flask import Flask, Request, Response, session
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncQueryTokenException(Exception):
+ pass
+
+
+class AsyncQueryJobException(Exception):
+ pass
+
+
+def build_job_metadata(channel_id: str, job_id: str, **kwargs: Any) ->
Dict[str, Any]:
+ return {
+ "channel_id": channel_id,
+ "job_id": job_id,
+ "user_id": session.get("user_id"),
+ "status": kwargs.get("status"),
+ "errors": kwargs.get("errors", []),
+ "result_url": kwargs.get("result_url"),
+ }
+
+
+def parse_event(event_data: Tuple[str, Dict[str, Any]]) -> Dict[str, Any]:
+ event_id = event_data[0]
+ event_payload = event_data[1]["data"]
+ return {"id": event_id, **json.loads(event_payload)}
+
+
+class AsyncQueryManager:
+ MAX_EVENT_COUNT = 100
+ STATUS_PENDING = "pending"
+ STATUS_RUNNING = "running"
+ STATUS_ERROR = "error"
+ STATUS_DONE = "done"
+
+ def __init__(self) -> None:
+ super().__init__()
+ self._redis: redis.Redis
+ self._stream_prefix: str = ""
+ self._stream_limit: Optional[int]
+ self._stream_limit_firehose: Optional[int]
+ self._jwt_cookie_name: str
+ self._jwt_cookie_secure: bool = False
+ self._jwt_secret: str
+
+ def init_app(self, app: Flask) -> None:
+ config = app.config
+ if len(config.get("GLOBAL_ASYNC_QUERIES_JWT_SECRET", "")) < 32:
Review comment:
Nit: config params should always be assumed as set (as they are here),
i.e. can be retrieved with `config["GLOBAL_ASYNC_QUERIES_JWT_SECRET"]`
##########
File path: superset-frontend/src/middleware/asyncEvent.ts
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { Middleware, MiddlewareAPI, Dispatch } from 'redux';
+import { SupersetClient } from '@superset-ui/core';
+import { SupersetError } from 'src/components/ErrorMessage/types';
+import { isFeatureEnabled, FeatureFlag } from '../featureFlags';
+import {
+ getClientErrorObject,
+ parseErrorJson,
+} from '../utils/getClientErrorObject';
+
+export type AsyncEvent = {
+ id: string;
+ channel_id: string;
+ job_id: string;
+ user_id: string;
+ status: string;
+ errors: SupersetError[];
+ result_url: string;
+};
+
+type AsyncEventOptions = {
+ getPendingComponents: (state: any) => any[];
+ successAction: (componentId: number, componentData: any) => { type: string };
+ errorAction: (componentId: number, response: any) => { type: string };
+ processEventsCallback?: (events: AsyncEvent[]) => void; // this is currently
used only for tests
+};
+
+type CachedDataResponse = {
+ componentId: number;
+ status: string;
+ data: any;
+};
+
+const initAsyncEvents = (options: AsyncEventOptions) => {
+ const POLLING_DELAY = 250;
Review comment:
Should we move this to `superset-frontend/src/constants.ts`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
