This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new c4d2238a16 feat: add Spark Sql DB engine spec and support Spark 3.x
(#20462)
c4d2238a16 is described below
commit c4d2238a1629851b0c044dba3f20c3af4c0a9031
Author: Susur <[email protected]>
AuthorDate: Tue Jun 28 06:50:35 2022 +0800
feat: add Spark Sql DB engine spec and support Spark 3.x (#20462)
* add apache spark3
* add Spark DB engine spec
* rebase secret key
* modify License error
* rebase databricks
* modify code style
* black code style
Co-authored-by: kai.he01 <[email protected]>
---
setup.py | 1 +
superset/db_engine_specs/databricks.py | 2 +-
superset/db_engine_specs/hive.py | 5 ---
.../db_engine_specs/{databricks.py => spark.py} | 38 ++--------------------
4 files changed, 5 insertions(+), 41 deletions(-)
diff --git a/setup.py b/setup.py
index bf5e06fed6..9e8f34fb8b 100644
--- a/setup.py
+++ b/setup.py
@@ -165,6 +165,7 @@ setup(
"snowflake": [
"snowflake-sqlalchemy==1.2.4"
], # PINNED! 1.2.5 introduced breaking changes requiring
sqlalchemy>=1.4.0
+ "spark": ["pyhive[hive]>=0.6.5", "tableschema", "thrift>=0.11.0,
<1.0.0"],
"teradata": ["teradatasql>=16.20.0.23"],
"thumbnails": ["Pillow>=9.0.1, <10.0.0"],
"vertica": ["sqlalchemy-vertica-python>=0.5.9, < 0.6"],
diff --git a/superset/db_engine_specs/databricks.py
b/superset/db_engine_specs/databricks.py
index d010b520d0..8803d0522e 100644
--- a/superset/db_engine_specs/databricks.py
+++ b/superset/db_engine_specs/databricks.py
@@ -13,7 +13,7 @@
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
-# under the License.o
+# under the License.
from datetime import datetime
from typing import Any, Dict, Optional
diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py
index 344d1f7ae9..73cc696d4b 100644
--- a/superset/db_engine_specs/hive.py
+++ b/superset/db_engine_specs/hive.py
@@ -580,8 +580,3 @@ class HiveEngineSpec(PrestoEngineSpec):
"""
return True
-
-
-class SparkEngineSpec(HiveEngineSpec):
-
- engine_name = "Apache Spark SQL"
diff --git a/superset/db_engine_specs/databricks.py
b/superset/db_engine_specs/spark.py
similarity index 59%
copy from superset/db_engine_specs/databricks.py
copy to superset/db_engine_specs/spark.py
index d010b520d0..a6eeb2e9db 100644
--- a/superset/db_engine_specs/databricks.py
+++ b/superset/db_engine_specs/spark.py
@@ -13,12 +13,8 @@
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
-# under the License.o
+# under the License.
-from datetime import datetime
-from typing import Any, Dict, Optional
-
-from superset.db_engine_specs.base import BaseEngineSpec
from superset.db_engine_specs.hive import HiveEngineSpec
time_grain_expressions = {
@@ -40,34 +36,6 @@ time_grain_expressions = {
}
-class DatabricksHiveEngineSpec(HiveEngineSpec):
- engine = "databricks"
- engine_name = "Databricks Interactive Cluster"
- driver = "pyhive"
- _show_functions_column = "function"
-
- _time_grain_expressions = time_grain_expressions
-
-
-class DatabricksODBCEngineSpec(BaseEngineSpec):
- engine = "databricks"
- engine_name = "Databricks SQL Endpoint"
- driver = "pyodbc"
-
+class SparkEngineSpec(HiveEngineSpec):
_time_grain_expressions = time_grain_expressions
-
- @classmethod
- def convert_dttm(
- cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str,
Any]] = None
- ) -> Optional[str]:
- return HiveEngineSpec.convert_dttm(target_type, dttm,
db_extra=db_extra)
-
- @classmethod
- def epoch_to_dttm(cls) -> str:
- return HiveEngineSpec.epoch_to_dttm()
-
-
-class DatabricksNativeEngineSpec(DatabricksODBCEngineSpec):
- engine = "databricks"
- engine_name = "Databricks Native Connector"
- driver = "connector"
+ engine_name = "Apache Spark SQL"
