This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new 633355a feat: export charts as ZIP files (#11349)
633355a is described below
commit 633355ab000823a0a8e1f8737c91db036517eaec
Author: Beto Dealmeida <[email protected]>
AuthorDate: Thu Oct 22 12:06:58 2020 -0700
feat: export charts as ZIP files (#11349)
* Export datasets as ZIP files
* Add logging when failing to parse extra
* Export charts as Zip file
* Fix lint
---
superset/charts/api.py | 66 +++++++++++++++++++++++-
superset/charts/commands/export.py | 83 ++++++++++++++++++++++++++++++
superset/charts/schemas.py | 1 +
superset/datasets/api.py | 2 +-
tests/charts/api_tests.py | 43 ++++++++++++++++
tests/charts/commands_tests.py | 101 +++++++++++++++++++++++++++++++++++++
6 files changed, 294 insertions(+), 2 deletions(-)
diff --git a/superset/charts/api.py b/superset/charts/api.py
index cdf633e..3ef27d0 100644
--- a/superset/charts/api.py
+++ b/superset/charts/api.py
@@ -16,10 +16,13 @@
# under the License.
import json
import logging
+from datetime import datetime
+from io import BytesIO
from typing import Any, Dict
+from zipfile import ZipFile
import simplejson
-from flask import g, make_response, redirect, request, Response, url_for
+from flask import g, make_response, redirect, request, Response, send_file,
url_for
from flask_appbuilder.api import expose, protect, rison, safe
from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_babel import gettext as _, ngettext
@@ -40,6 +43,7 @@ from superset.charts.commands.exceptions import (
ChartNotFoundError,
ChartUpdateFailedError,
)
+from superset.charts.commands.export import ExportChartsCommand
from superset.charts.commands.update import UpdateChartCommand
from superset.charts.filters import ChartAllTextFilter, ChartFavoriteFilter,
ChartFilter
from superset.charts.schemas import (
@@ -48,6 +52,7 @@ from superset.charts.schemas import (
ChartPostSchema,
ChartPutSchema,
get_delete_ids_schema,
+ get_export_ids_schema,
openapi_spec_methods_override,
screenshot_query_schema,
thumbnail_query_schema,
@@ -709,3 +714,62 @@ class ChartRestApi(BaseSupersetModelRestApi):
return Response(
FileWrapper(screenshot), mimetype="image/png",
direct_passthrough=True
)
+
+ @expose("/export/", methods=["GET"])
+ @protect()
+ @safe
+ @statsd_metrics
+ @rison(get_export_ids_schema)
+ def export(self, **kwargs: Any) -> Response:
+ """Export charts
+ ---
+ get:
+ description: >-
+ Exports multiple charts and downloads them as YAML files
+ parameters:
+ - in: query
+ name: q
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ type: integer
+ responses:
+ 200:
+ description: A zip file with chart(s), dataset(s) and
database(s) as YAML
+ content:
+ application/zip:
+ schema:
+ type: string
+ format: binary
+ 400:
+ $ref: '#/components/responses/400'
+ 401:
+ $ref: '#/components/responses/401'
+ 404:
+ $ref: '#/components/responses/404'
+ 500:
+ $ref: '#/components/responses/500'
+ """
+ requested_ids = kwargs["rison"]
+ timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
+ root = f"chart_export_{timestamp}"
+ filename = f"{root}.zip"
+
+ buf = BytesIO()
+ with ZipFile(buf, "w") as bundle:
+ try:
+ for file_name, file_content in
ExportChartsCommand(requested_ids).run():
+ with bundle.open(f"{root}/{file_name}", "w") as fp:
+ fp.write(file_content.encode())
+ except ChartNotFoundError:
+ return self.response_404()
+ buf.seek(0)
+
+ return send_file(
+ buf,
+ mimetype="application/zip",
+ as_attachment=True,
+ attachment_filename=filename,
+ )
diff --git a/superset/charts/commands/export.py
b/superset/charts/commands/export.py
new file mode 100644
index 0000000..00e0fd4
--- /dev/null
+++ b/superset/charts/commands/export.py
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# isort:skip_file
+
+import json
+from typing import Iterator, List, Tuple
+
+import yaml
+
+from superset.commands.base import BaseCommand
+from superset.charts.commands.exceptions import ChartNotFoundError
+from superset.charts.dao import ChartDAO
+from superset.datasets.commands.export import ExportDatasetsCommand
+from superset.utils.dict_import_export import IMPORT_EXPORT_VERSION, sanitize
+from superset.models.slice import Slice
+
+
+# keys present in the standard export that are not needed
+REMOVE_KEYS = ["datasource_type", "datasource_name"]
+
+
+class ExportChartsCommand(BaseCommand):
+ def __init__(self, chart_ids: List[int]):
+ self.chart_ids = chart_ids
+
+ # this will be set when calling validate()
+ self._models: List[Slice] = []
+
+ @staticmethod
+ def export_chart(chart: Slice) -> Iterator[Tuple[str, str]]:
+ chart_slug = sanitize(chart.slice_name)
+ file_name = f"charts/{chart_slug}.yaml"
+
+ payload = chart.export_to_dict(
+ recursive=False,
+ include_parent_ref=False,
+ include_defaults=True,
+ export_uuids=True,
+ )
+ # TODO (betodealmeida): move this logic to export_to_dict once this
+ # becomes the default export endpoint
+ for key in REMOVE_KEYS:
+ del payload[key]
+ if "params" in payload:
+ try:
+ payload["params"] = json.loads(payload["params"])
+ except json.decoder.JSONDecodeError:
+ pass
+
+ payload["version"] = IMPORT_EXPORT_VERSION
+ if chart.table:
+ payload["dataset_uuid"] = str(chart.table.uuid)
+
+ file_content = yaml.safe_dump(payload, sort_keys=False)
+ yield file_name, file_content
+
+ if chart.table:
+ yield from ExportDatasetsCommand([chart.table.id]).run()
+
+ def run(self) -> Iterator[Tuple[str, str]]:
+ self.validate()
+
+ for chart in self._models:
+ yield from self.export_chart(chart)
+
+ def validate(self) -> None:
+ self._models = ChartDAO.find_by_ids(self.chart_ids)
+ if len(self._models) != len(self.chart_ids):
+ raise ChartNotFoundError()
diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py
index 58012a8..f54621c 100644
--- a/superset/charts/schemas.py
+++ b/superset/charts/schemas.py
@@ -45,6 +45,7 @@ screenshot_query_schema = {
"thumb_size": width_height_schema,
},
}
+get_export_ids_schema = {"type": "array", "items": {"type": "integer"}}
#
# Column schema descriptions
diff --git a/superset/datasets/api.py b/superset/datasets/api.py
index 2ed051f..fff69bf 100644
--- a/superset/datasets/api.py
+++ b/superset/datasets/api.py
@@ -347,7 +347,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
@statsd_metrics
@rison(get_export_ids_schema)
def export(self, **kwargs: Any) -> Response:
- """Export dashboards
+ """Export datasets
---
get:
description: >-
diff --git a/tests/charts/api_tests.py b/tests/charts/api_tests.py
index c7023a8..4d3a67e 100644
--- a/tests/charts/api_tests.py
+++ b/tests/charts/api_tests.py
@@ -19,7 +19,9 @@
import json
from typing import List, Optional
from datetime import datetime
+from io import BytesIO
from unittest import mock
+from zipfile import is_zipfile
import humanize
import prison
@@ -1073,3 +1075,44 @@ class TestChartApi(SupersetTestCase,
ApiOwnersTestCaseMixin):
result = response_payload["result"][0]["query"]
if get_example_database().backend != "presto":
assert "('boy' = 'boy')" in result
+
+ def test_export_chart(self):
+ """
+ Chart API: Test export dataset
+ """
+ example_chart = db.session.query(Slice).all()[0]
+ argument = [example_chart.id]
+ uri = f"api/v1/chart/export/?q={prison.dumps(argument)}"
+
+ self.login(username="admin")
+ rv = self.get_assert_metric(uri, "export")
+
+ assert rv.status_code == 200
+
+ buf = BytesIO(rv.data)
+ assert is_zipfile(buf)
+
+ def test_export_chart_not_found(self):
+ """
+ Dataset API: Test export dataset not found
+ """
+ # Just one does not exist and we get 404
+ argument = [-1, 1]
+ uri = f"api/v1/chart/export/?q={prison.dumps(argument)}"
+ self.login(username="admin")
+ rv = self.get_assert_metric(uri, "export")
+
+ assert rv.status_code == 404
+
+ def test_export_chart_gamma(self):
+ """
+ Dataset API: Test export dataset has gamma
+ """
+ example_chart = db.session.query(Slice).all()[0]
+ argument = [example_chart.id]
+ uri = f"api/v1/chart/export/?q={prison.dumps(argument)}"
+
+ self.login(username="gamma")
+ rv = self.client.get(uri)
+
+ assert rv.status_code == 404
diff --git a/tests/charts/commands_tests.py b/tests/charts/commands_tests.py
new file mode 100644
index 0000000..6923757
--- /dev/null
+++ b/tests/charts/commands_tests.py
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest.mock import patch
+
+import yaml
+
+from superset import db, security_manager
+from superset.charts.commands.exceptions import ChartNotFoundError
+from superset.charts.commands.export import ExportChartsCommand
+from superset.models.slice import Slice
+from tests.base_tests import SupersetTestCase
+
+
+class TestExportChartsCommand(SupersetTestCase):
+ @patch("superset.security.manager.g")
+ def test_export_chart_command(self, mock_g):
+ mock_g.user = security_manager.find_user("admin")
+
+ example_chart = db.session.query(Slice).all()[0]
+ command = ExportChartsCommand(chart_ids=[example_chart.id])
+ contents = dict(command.run())
+
+ expected = [
+ "charts/energy_sankey.yaml",
+ "datasets/examples/energy_usage.yaml",
+ "databases/examples.yaml",
+ ]
+ assert expected == list(contents.keys())
+
+ metadata = yaml.safe_load(contents["charts/energy_sankey.yaml"])
+ assert metadata == {
+ "slice_name": "Energy Sankey",
+ "viz_type": "sankey",
+ "params": {
+ "collapsed_fieldsets": "",
+ "groupby": ["source", "target",],
+ "metric": "sum__value",
+ "row_limit": "5000",
+ "slice_name": "Energy Sankey",
+ "viz_type": "sankey",
+ },
+ "cache_timeout": None,
+ "dataset_uuid": str(example_chart.table.uuid),
+ "uuid": str(example_chart.uuid),
+ "version": "1.0.0",
+ }
+
+ @patch("superset.security.manager.g")
+ def test_export_chart_command_no_access(self, mock_g):
+ """Test that users can't export datasets they don't have access to"""
+ mock_g.user = security_manager.find_user("gamma")
+
+ example_chart = db.session.query(Slice).all()[0]
+ command = ExportChartsCommand(chart_ids=[example_chart.id])
+ contents = command.run()
+ with self.assertRaises(ChartNotFoundError):
+ next(contents)
+
+ @patch("superset.security.manager.g")
+ def test_export_chart_command_invalid_dataset(self, mock_g):
+ """Test that an error is raised when exporting an invalid dataset"""
+ mock_g.user = security_manager.find_user("admin")
+ command = ExportChartsCommand(chart_ids=[-1])
+ contents = command.run()
+ with self.assertRaises(ChartNotFoundError):
+ next(contents)
+
+ @patch("superset.security.manager.g")
+ def test_export_chart_command_key_order(self, mock_g):
+ """Test that they keys in the YAML have the same order as
export_fields"""
+ mock_g.user = security_manager.find_user("admin")
+
+ example_chart = db.session.query(Slice).all()[0]
+ command = ExportChartsCommand(chart_ids=[example_chart.id])
+ contents = dict(command.run())
+
+ metadata = yaml.safe_load(contents["charts/energy_sankey.yaml"])
+ assert list(metadata.keys()) == [
+ "slice_name",
+ "viz_type",
+ "params",
+ "cache_timeout",
+ "uuid",
+ "version",
+ "dataset_uuid",
+ ]
