This is an automated email from the ASF dual-hosted git repository.
erikrit pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new b686004 feat: add configuration for Presto cursor poll interval
(#10191)
b686004 is described below
commit b686004294326f69a8aa8e587881a4747d110f0d
Author: Erik Ritter <erik.rit...@airbnb.com>
AuthorDate: Tue Jun 30 08:56:22 2020 -0700
feat: add configuration for Presto cursor poll interval (#10191)
---
superset/db_engine_specs/presto.py | 8 +++++++-
superset/models/core.py | 4 ++++
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/superset/db_engine_specs/presto.py
b/superset/db_engine_specs/presto.py
index 6c014a5..ebf8ec4 100644
--- a/superset/db_engine_specs/presto.py
+++ b/superset/db_engine_specs/presto.py
@@ -51,6 +51,9 @@ QueryStatus = utils.QueryStatus
config = app.config
logger = logging.getLogger(__name__)
+# See here:
https://github.com/dropbox/PyHive/blob/8eb0aeab8ca300f3024655419b93dad926c1a351/pyhive/presto.py#L93
# pylint: disable=line-too-long
+DEFAULT_PYHIVE_POLL_INTERVAL = 1
+
def get_children(column: Dict[str, str]) -> List[Dict[str, str]]:
"""
@@ -729,6 +732,9 @@ class PrestoEngineSpec(BaseEngineSpec):
def handle_cursor(cls, cursor: Any, query: Query, session: Session) ->
None:
"""Updates progress information"""
query_id = query.id
+ poll_interval = query.database.connect_args.get(
+ "poll_interval", DEFAULT_PYHIVE_POLL_INTERVAL
+ )
logger.info("Query %i: Polling the cursor for progress", query_id)
polled = cursor.poll()
# poll returns dict -- JSON status information or ``None``
@@ -762,7 +768,7 @@ class PrestoEngineSpec(BaseEngineSpec):
if progress > query.progress:
query.progress = progress
session.commit()
- time.sleep(1)
+ time.sleep(poll_interval)
logger.info("Query %i: Polling the cursor for progress", query_id)
polled = cursor.poll()
diff --git a/superset/models/core.py b/superset/models/core.py
index 595cbc7..7660150 100755
--- a/superset/models/core.py
+++ b/superset/models/core.py
@@ -249,6 +249,10 @@ class Database(
def default_schemas(self) -> List[str]:
return self.get_extra().get("default_schemas", [])
+ @property
+ def connect_args(self) -> Dict[str, Any]:
+ return self.get_extra().get("engine_params", {}).get("connect_args",
{})
+
@classmethod
def get_password_masked_url_from_uri( # pylint: disable=invalid-name
cls, uri: str
