This is an automated email from the ASF dual-hosted git repository. beto pushed a commit to branch import-encrypted-extra in repository https://gitbox.apache.org/repos/asf/superset.git
commit e8e673c7d0692b40585d127ad6950613ce917dcc Author: Beto Dealmeida <[email protected]> AuthorDate: Thu Feb 20 20:54:37 2025 -0600 feat: allow importing encrypted_extra --- superset/databases/schemas.py | 4 +- .../commands/databases/importers/__init__.py | 16 ++++++ .../commands/databases/importers/v1/__init__.py | 16 ++++++ .../databases/importers/v1/command_test.py | 50 ++++++++++++++++++ tests/unit_tests/databases/api_test.py | 59 ++++++++++++++++++++++ 5 files changed, 143 insertions(+), 2 deletions(-) diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py index 8312a7a1b2..499383a402 100644 --- a/superset/databases/schemas.py +++ b/superset/databases/schemas.py @@ -229,7 +229,7 @@ def server_cert_validator(value: str) -> str: return value -def encrypted_extra_validator(value: str) -> str: +def encrypted_extra_validator(value: str | None) -> None: """ Validate that encrypted extra is a valid JSON string """ @@ -240,7 +240,6 @@ def encrypted_extra_validator(value: str) -> str: raise ValidationError( [_("Field cannot be decoded by JSON. %(msg)s", msg=str(ex))] ) from ex - return value def extra_validator(value: str) -> str: @@ -855,6 +854,7 @@ class ImportV1DatabaseSchema(Schema): database_name = fields.String(required=True) sqlalchemy_uri = fields.String(required=True) password = fields.String(allow_none=True) + encrypted_extra = fields.String(allow_none=True, validate=encrypted_extra_validator) cache_timeout = fields.Integer(allow_none=True) expose_in_sqllab = fields.Boolean() allow_run_async = fields.Boolean() diff --git a/tests/unit_tests/commands/databases/importers/__init__.py b/tests/unit_tests/commands/databases/importers/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/tests/unit_tests/commands/databases/importers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/commands/databases/importers/v1/__init__.py b/tests/unit_tests/commands/databases/importers/v1/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/tests/unit_tests/commands/databases/importers/v1/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/commands/databases/importers/v1/command_test.py b/tests/unit_tests/commands/databases/importers/v1/command_test.py new file mode 100644 index 0000000000..1243d57b5b --- /dev/null +++ b/tests/unit_tests/commands/databases/importers/v1/command_test.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import copy +import json + +from pytest_mock import MockerFixture +from sqlalchemy.orm.session import Session + +from tests.unit_tests.fixtures.assets_configs import databases_config + + +def test_import_database_with_encrypted_extra( + mocker: MockerFixture, + session: Session, +) -> None: + """ + Test that databases are imported with their encrypted extra info when available. + """ + from superset import db, security_manager + from superset.commands.database.importers.v1 import ImportDatabasesCommand + from superset.models.core import Database + + mocker.patch.object(security_manager, "can_access", return_value=True) + + engine = db.session.get_bind() + Database.metadata.create_all(engine) # pylint: disable=no-member + configs = copy.deepcopy(databases_config) + configs["databases/examples.yaml"]["encrypted_extra"] = json.dumps( + {"secret": "info"}, + ) + + ImportDatabasesCommand._import(configs) + uuid = configs["databases/examples.yaml"]["uuid"] + database = db.session.query(Database).filter_by(uuid=uuid).one() + assert database.encrypted_extra == '{"secret": "info"}' diff --git a/tests/unit_tests/databases/api_test.py b/tests/unit_tests/databases/api_test.py index 63d61ce82a..f6eae6544d 100644 --- a/tests/unit_tests/databases/api_test.py +++ b/tests/unit_tests/databases/api_test.py @@ -26,6 +26,7 @@ from unittest.mock import ANY, Mock from uuid import UUID import pytest +import yaml from flask import current_app from freezegun import freeze_time from pytest_mock import MockerFixture @@ -377,6 +378,64 @@ def test_update_with_password_mask( ) +def test_import( + mocker: MockerFixture, + client: Any, + full_api_access: None, +) -> None: + """ + Test that we can import a database export. + """ + contents = { + "metadata.yaml": yaml.safe_dump( + { + "version": "1.0.0", + "type": "Database", + "timestamp": "2021-01-01T00:00:00Z", + } + ), + "databases/test.yaml": yaml.safe_dump( + { + "database_name": "test", + "sqlalchemy_uri": "bigquery://gcp-project-id/", + "cache_timeout": 0, + "expose_in_sqllab": True, + "allow_run_async": False, + "allow_ctas": False, + "allow_cvas": False, + "allow_dml": False, + "allow_file_upload": False, + "encrypted_extra": json.dumps({"secret": "info"}), + "extra": json.dumps({"allows_virtual_table_explore": True}), + "uuid": "00000000-0000-0000-0000-123456789001", + } + ), + } + mocker.patch("superset.databases.api.is_zipfile", return_value=True) + mocker.patch("superset.databases.api.ZipFile") + mocker.patch( + "superset.databases.api.get_contents_from_bundle", + return_value=contents, + ) + command = mocker.patch("superset.databases.api.ImportDatabasesCommand") + + form_data = {"formData": (BytesIO(b"test"), "test.zip")} + client.post( + "/api/v1/database/import/", + data=form_data, + content_type="multipart/form-data", + ) + + command.assert_called_with( + contents, + passwords=None, + overwrite=False, + ssh_tunnel_passwords=None, + ssh_tunnel_private_keys=None, + ssh_tunnel_priv_key_passwords=None, + ) + + def test_non_zip_import(client: Any, full_api_access: None) -> None: """ Test that non-ZIP imports are not allowed.
