This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new b05e7db feat: API for asset sync (#19220)
b05e7db is described below
commit b05e7dbf2aa513cd50a20847a858625916579dcb
Author: Beto Dealmeida <[email protected]>
AuthorDate: Tue Mar 22 08:29:24 2022 -0700
feat: API for asset sync (#19220)
* feat: API for asset sync
* Add unit tests.
* Improve tests
* Move files
* Add more tests
---
superset/commands/importers/exceptions.py | 5 +
superset/importexport/api.py | 163 +++++++++++++
superset/initialization/__init__.py | 4 +-
tests/unit_tests/conftest.py | 46 ++--
.../unit_tests/importexport/__init__.py | 12 -
tests/unit_tests/importexport/api_test.py | 254 +++++++++++++++++++++
.../unit_tests/views/__init__.py | 12 -
7 files changed, 458 insertions(+), 38 deletions(-)
diff --git a/superset/commands/importers/exceptions.py
b/superset/commands/importers/exceptions.py
index c1beb8e..54388bd 100644
--- a/superset/commands/importers/exceptions.py
+++ b/superset/commands/importers/exceptions.py
@@ -26,3 +26,8 @@ class IncorrectVersionError(CommandException):
class NoValidFilesFoundError(CommandException):
status = 400
message = "No valid import files were found"
+
+
+class IncorrectFormatError(CommandException):
+ status = 422
+ message = "File has the incorrect format"
diff --git a/superset/importexport/api.py b/superset/importexport/api.py
new file mode 100644
index 0000000..156b4c2
--- /dev/null
+++ b/superset/importexport/api.py
@@ -0,0 +1,163 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+from datetime import datetime
+from io import BytesIO
+from zipfile import is_zipfile, ZipFile
+
+from flask import request, Response, send_file
+from flask_appbuilder.api import BaseApi, expose, protect
+
+from superset.commands.export.assets import ExportAssetsCommand
+from superset.commands.importers.exceptions import (
+ IncorrectFormatError,
+ NoValidFilesFoundError,
+)
+from superset.commands.importers.v1.assets import ImportAssetsCommand
+from superset.commands.importers.v1.utils import get_contents_from_bundle
+from superset.extensions import event_logger
+from superset.views.base_api import requires_form_data
+
+
+class ImportExportRestApi(BaseApi):
+ """
+ API for exporting all assets or importing them.
+ """
+
+ resource_name = "assets"
+ openapi_spec_tag = "Import/export"
+
+ @expose("/export/", methods=["GET"])
+ @protect()
+ @event_logger.log_this_with_context(
+ action=lambda self, *args, **kwargs:
f"{self.__class__.__name__}.export",
+ log_to_statsd=False,
+ )
+ def export(self) -> Response:
+ """
+ Export all assets.
+ ---
+ get:
+ description: >-
+ Returns a ZIP file with all the Superset assets (databases,
datasets, charts,
+ dashboards, saved queries) as YAML files.
+ responses:
+ 200:
+ description: ZIP file
+ content:
+ application/zip:
+ schema:
+ type: string
+ format: binary
+ 400:
+ $ref: '#/components/responses/400'
+ 401:
+ $ref: '#/components/responses/401'
+ 404:
+ $ref: '#/components/responses/404'
+ 500:
+ $ref: '#/components/responses/500'
+ """
+ timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
+ root = f"assets_export_{timestamp}"
+ filename = f"{root}.zip"
+
+ buf = BytesIO()
+ with ZipFile(buf, "w") as bundle:
+ for file_name, file_content in ExportAssetsCommand().run():
+ with bundle.open(f"{root}/{file_name}", "w") as fp:
+ fp.write(file_content.encode())
+ buf.seek(0)
+
+ response = send_file(
+ buf,
+ mimetype="application/zip",
+ as_attachment=True,
+ attachment_filename=filename,
+ )
+ return response
+
+ @expose("/import/", methods=["POST"])
+ @protect()
+ @event_logger.log_this_with_context(
+ action=lambda self, *args, **kwargs:
f"{self.__class__.__name__}.import_",
+ log_to_statsd=False,
+ )
+ @requires_form_data
+ def import_(self) -> Response:
+ """Import multiple assets
+ ---
+ post:
+ requestBody:
+ required: true
+ content:
+ multipart/form-data:
+ schema:
+ type: object
+ properties:
+ bundle:
+ description: upload file (ZIP or JSON)
+ type: string
+ format: binary
+ passwords:
+ description: >-
+ JSON map of passwords for each featured database in the
+ ZIP file. If the ZIP includes a database config in the
path
+ `databases/MyDatabase.yaml`, the password should be
provided
+ in the following format:
+ `{"databases/MyDatabase.yaml": "my_password"}`.
+ type: string
+ responses:
+ 200:
+ description: Dashboard import result
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ message:
+ type: string
+ 400:
+ $ref: '#/components/responses/400'
+ 401:
+ $ref: '#/components/responses/401'
+ 422:
+ $ref: '#/components/responses/422'
+ 500:
+ $ref: '#/components/responses/500'
+ """
+ upload = request.files.get("bundle")
+ if not upload:
+ return self.response_400()
+ if not is_zipfile(upload):
+ raise IncorrectFormatError("Not a ZIP file")
+
+ with ZipFile(upload) as bundle:
+ contents = get_contents_from_bundle(bundle)
+
+ if not contents:
+ raise NoValidFilesFoundError()
+
+ passwords = (
+ json.loads(request.form["passwords"])
+ if "passwords" in request.form
+ else None
+ )
+
+ command = ImportAssetsCommand(contents, passwords=passwords)
+ command.run()
+ return self.response(200, message="OK")
diff --git a/superset/initialization/__init__.py
b/superset/initialization/__init__.py
index 6e2d927..f6ffd3e 100644
--- a/superset/initialization/__init__.py
+++ b/superset/initialization/__init__.py
@@ -67,7 +67,7 @@ class SupersetAppInitializer: # pylint:
disable=too-many-public-methods
self.config = app.config
self.manifest: Dict[Any, Any] = {}
- @deprecated(details="use self.superset_app instead of self.flask_app") #
type: ignore # pylint: disable=line-too-long,useless-suppression
+ @deprecated(details="use self.superset_app instead of self.flask_app") #
type: ignore
@property
def flask_app(self) -> SupersetApp:
return self.superset_app
@@ -143,6 +143,7 @@ class SupersetAppInitializer: # pylint:
disable=too-many-public-methods
from superset.datasets.metrics.api import DatasetMetricRestApi
from superset.explore.form_data.api import ExploreFormDataRestApi
from superset.explore.permalink.api import ExplorePermalinkRestApi
+ from superset.importexport.api import ImportExportRestApi
from superset.queries.api import QueryRestApi
from superset.queries.saved_queries.api import SavedQueryRestApi
from superset.reports.api import ReportScheduleRestApi
@@ -219,6 +220,7 @@ class SupersetAppInitializer: # pylint:
disable=too-many-public-methods
appbuilder.add_api(ExploreFormDataRestApi)
appbuilder.add_api(ExplorePermalinkRestApi)
appbuilder.add_api(FilterSetRestApi)
+ appbuilder.add_api(ImportExportRestApi)
appbuilder.add_api(QueryRestApi)
appbuilder.add_api(ReportScheduleRestApi)
appbuilder.add_api(ReportExecutionLogRestApi)
diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py
index 8522877..4987aaf 100644
--- a/tests/unit_tests/conftest.py
+++ b/tests/unit_tests/conftest.py
@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# pylint: disable=redefined-outer-name
+# pylint: disable=redefined-outer-name, import-outside-toplevel
-from typing import Iterator
+import importlib
+from typing import Any, Iterator
import pytest
from pytest_mock import MockFixture
@@ -25,11 +26,12 @@ from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.session import Session
from superset.app import SupersetApp
+from superset.extensions import appbuilder
from superset.initialization import SupersetAppInitializer
[email protected]()
-def session() -> Iterator[Session]:
[email protected]
+def session(mocker: MockFixture) -> Iterator[Session]:
"""
Create an in-memory SQLite session to test models.
"""
@@ -40,11 +42,18 @@ def session() -> Iterator[Session]:
# flask calls session.remove()
in_memory_session.remove = lambda: None
+ # patch session
+ mocker.patch(
+ "superset.security.SupersetSecurityManager.get_session",
+ return_value=in_memory_session,
+ )
+ mocker.patch("superset.db.session", in_memory_session)
+
yield in_memory_session
[email protected]
-def app(mocker: MockFixture, session: Session) -> Iterator[SupersetApp]:
[email protected](scope="module")
+def app() -> Iterator[SupersetApp]:
"""
A fixture that generates a Superset app.
"""
@@ -52,21 +61,32 @@ def app(mocker: MockFixture, session: Session) ->
Iterator[SupersetApp]:
app.config.from_object("superset.config")
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite://"
- app.config["FAB_ADD_SECURITY_VIEWS"] = False
+ app.config["TESTING"] = True
- app_initializer = app.config.get("APP_INITIALIZER",
SupersetAppInitializer)(app)
+ # ``superset.extensions.appbuilder`` is a singleton, and won't rebuild the
+ # routes when this fixture is called multiple times; we need to clear the
+ # registered views to ensure the initialization can happen more than once.
+ appbuilder.baseviews = []
+
+ app_initializer = SupersetAppInitializer(app)
app_initializer.init_app()
- # patch session
- mocker.patch(
- "superset.security.SupersetSecurityManager.get_session",
return_value=session,
- )
- mocker.patch("superset.db.session", session)
+ # reload base views to ensure error handlers are applied to the app
+ with app.app_context():
+ import superset.views.base
+
+ importlib.reload(superset.views.base)
yield app
@pytest.fixture
+def client(app: SupersetApp) -> Any:
+ with app.test_client() as client:
+ yield client
+
+
[email protected]
def app_context(app: SupersetApp) -> Iterator[None]:
"""
A fixture that yields and application context.
diff --git a/superset/commands/importers/exceptions.py
b/tests/unit_tests/importexport/__init__.py
similarity index 73%
copy from superset/commands/importers/exceptions.py
copy to tests/unit_tests/importexport/__init__.py
index c1beb8e..13a8339 100644
--- a/superset/commands/importers/exceptions.py
+++ b/tests/unit_tests/importexport/__init__.py
@@ -14,15 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from superset.commands.exceptions import CommandException
-
-
-class IncorrectVersionError(CommandException):
- status = 422
- message = "Import has incorrect version"
-
-
-class NoValidFilesFoundError(CommandException):
- status = 400
- message = "No valid import files were found"
diff --git a/tests/unit_tests/importexport/api_test.py
b/tests/unit_tests/importexport/api_test.py
new file mode 100644
index 0000000..e5dee97
--- /dev/null
+++ b/tests/unit_tests/importexport/api_test.py
@@ -0,0 +1,254 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=invalid-name, import-outside-toplevel
+
+import json
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+from zipfile import is_zipfile, ZipFile
+
+from pytest_mock import MockFixture
+
+from superset import security_manager
+
+
+def test_export_assets(mocker: MockFixture, client: Any) -> None:
+ """
+ Test exporting assets.
+ """
+ from superset.commands.importers.v1.utils import get_contents_from_bundle
+
+ # grant access
+ mocker.patch(
+ "flask_appbuilder.security.decorators.verify_jwt_in_request",
return_value=True
+ )
+ mocker.patch.object(security_manager, "has_access", return_value=True)
+
+ mocked_contents = [
+ (
+ "metadata.yaml",
+ "version: 1.0.0\ntype: assets\ntimestamp:
'2022-01-01T00:00:00+00:00'\n",
+ ),
+ ("databases/example.yaml", "<DATABASE CONTENTS>"),
+ ]
+
+ ExportAssetsCommand =
mocker.patch("superset.importexport.api.ExportAssetsCommand")
+ ExportAssetsCommand().run.return_value = mocked_contents[:]
+
+ response = client.get("/api/v1/assets/export/")
+ assert response.status_code == 200
+
+ buf = BytesIO(response.data)
+ assert is_zipfile(buf)
+
+ buf.seek(0)
+ with ZipFile(buf) as bundle:
+ contents = get_contents_from_bundle(bundle)
+ assert contents == dict(mocked_contents)
+
+
+def test_import_assets(mocker: MockFixture, client: Any) -> None:
+ """
+ Test importing assets.
+ """
+ # grant access
+ mocker.patch(
+ "flask_appbuilder.security.decorators.verify_jwt_in_request",
return_value=True
+ )
+ mocker.patch.object(security_manager, "has_access", return_value=True)
+
+ mocked_contents = {
+ "metadata.yaml": (
+ "version: 1.0.0\ntype: assets\ntimestamp:
'2022-01-01T00:00:00+00:00'\n"
+ ),
+ "databases/example.yaml": "<DATABASE CONTENTS>",
+ }
+
+ ImportAssetsCommand =
mocker.patch("superset.importexport.api.ImportAssetsCommand")
+
+ root = Path("assets_export")
+ buf = BytesIO()
+ with ZipFile(buf, "w") as bundle:
+ for path, contents in mocked_contents.items():
+ with bundle.open(str(root / path), "w") as fp:
+ fp.write(contents.encode())
+ buf.seek(0)
+
+ form_data = {
+ "bundle": (buf, "assets_export.zip"),
+ "passwords": json.dumps(
+ {"assets_export/databases/imported_database.yaml": "SECRET"}
+ ),
+ }
+ response = client.post(
+ "/api/v1/assets/import/", data=form_data,
content_type="multipart/form-data"
+ )
+ assert response.status_code == 200
+ assert response.json == {"message": "OK"}
+
+ passwords = {"assets_export/databases/imported_database.yaml": "SECRET"}
+ ImportAssetsCommand.assert_called_with(mocked_contents,
passwords=passwords)
+
+
+def test_import_assets_not_zip(mocker: MockFixture, client: Any) -> None:
+ """
+ Test error message when the upload is not a ZIP file.
+ """
+ # grant access
+ mocker.patch(
+ "flask_appbuilder.security.decorators.verify_jwt_in_request",
return_value=True
+ )
+ mocker.patch.object(security_manager, "has_access", return_value=True)
+
+ buf = BytesIO(b"definitely_not_a_zip_file")
+ form_data = {
+ "bundle": (buf, "broken.txt"),
+ }
+ response = client.post(
+ "/api/v1/assets/import/", data=form_data,
content_type="multipart/form-data"
+ )
+ assert response.status_code == 422
+ assert response.json == {
+ "errors": [
+ {
+ "message": "Not a ZIP file",
+ "error_type": "GENERIC_COMMAND_ERROR",
+ "level": "warning",
+ "extra": {
+ "issue_codes": [
+ {
+ "code": 1010,
+ "message": (
+ "Issue 1010 - Superset encountered an error
while "
+ "running a command."
+ ),
+ }
+ ]
+ },
+ }
+ ]
+ }
+
+
+def test_import_assets_no_form_data(mocker: MockFixture, client: Any) -> None:
+ """
+ Test error message when the upload has no form data.
+ """
+ # grant access
+ mocker.patch(
+ "flask_appbuilder.security.decorators.verify_jwt_in_request",
return_value=True
+ )
+ mocker.patch.object(security_manager, "has_access", return_value=True)
+
+ response = client.post("/api/v1/assets/import/", data="some_content")
+ assert response.status_code == 400
+ assert response.json == {
+ "errors": [
+ {
+ "message": "Request MIME type is not 'multipart/form-data'",
+ "error_type": "INVALID_PAYLOAD_FORMAT_ERROR",
+ "level": "error",
+ "extra": {
+ "issue_codes": [
+ {
+ "code": 1019,
+ "message": (
+ "Issue 1019 - The submitted payload has the
incorrect "
+ "format."
+ ),
+ }
+ ]
+ },
+ }
+ ]
+ }
+
+
+def test_import_assets_incorrect_form_data(mocker: MockFixture, client: Any)
-> None:
+ """
+ Test error message when the upload form data has the wrong key.
+ """
+ # grant access
+ mocker.patch(
+ "flask_appbuilder.security.decorators.verify_jwt_in_request",
return_value=True
+ )
+ mocker.patch.object(security_manager, "has_access", return_value=True)
+
+ buf = BytesIO(b"definitely_not_a_zip_file")
+ form_data = {
+ "wrong": (buf, "broken.txt"),
+ }
+ response = client.post(
+ "/api/v1/assets/import/", data=form_data,
content_type="multipart/form-data"
+ )
+ assert response.status_code == 400
+ assert response.json == {"message": "Arguments are not correct"}
+
+
+def test_import_assets_no_contents(mocker: MockFixture, client: Any) -> None:
+ """
+ Test error message when the ZIP bundle has no contents.
+ """
+ # grant access
+ mocker.patch(
+ "flask_appbuilder.security.decorators.verify_jwt_in_request",
return_value=True
+ )
+ mocker.patch.object(security_manager, "has_access", return_value=True)
+
+ mocked_contents = {
+ "README.txt": "Something is wrong",
+ }
+
+ root = Path("assets_export")
+ buf = BytesIO()
+ with ZipFile(buf, "w") as bundle:
+ for path, contents in mocked_contents.items():
+ with bundle.open(str(root / path), "w") as fp:
+ fp.write(contents.encode())
+ buf.seek(0)
+
+ form_data = {
+ "bundle": (buf, "assets_export.zip"),
+ "passwords": json.dumps(
+ {"assets_export/databases/imported_database.yaml": "SECRET"}
+ ),
+ }
+ response = client.post(
+ "/api/v1/assets/import/", data=form_data,
content_type="multipart/form-data"
+ )
+ assert response.status_code == 400
+ assert response.json == {
+ "errors": [
+ {
+ "message": "No valid import files were found",
+ "error_type": "GENERIC_COMMAND_ERROR",
+ "level": "warning",
+ "extra": {
+ "issue_codes": [
+ {
+ "code": 1010,
+ "message": (
+ "Issue 1010 - Superset encountered an error
while "
+ "running a command."
+ ),
+ }
+ ]
+ },
+ }
+ ]
+ }
diff --git a/superset/commands/importers/exceptions.py
b/tests/unit_tests/views/__init__.py
similarity index 73%
copy from superset/commands/importers/exceptions.py
copy to tests/unit_tests/views/__init__.py
index c1beb8e..13a8339 100644
--- a/superset/commands/importers/exceptions.py
+++ b/tests/unit_tests/views/__init__.py
@@ -14,15 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from superset.commands.exceptions import CommandException
-
-
-class IncorrectVersionError(CommandException):
- status = 422
- message = "Import has incorrect version"
-
-
-class NoValidFilesFoundError(CommandException):
- status = 400
- message = "No valid import files were found"
