This is an automated email from the ASF dual-hosted git repository.
fisjac pushed a commit to branch fisjac/bigquery-import-bug
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/fisjac/bigquery-import-bug by
this push:
new 54f044b20d WIP investigating problem and possible solutions
54f044b20d is described below
commit 54f044b20dba29743af224adc0b1b9ee850b8068
Author: Jack Fisher <[email protected]>
AuthorDate: Fri Sep 27 14:31:25 2024 -0500
WIP investigating problem and possible solutions
---
.../src/features/databases/DatabaseModal/index.tsx | 4 ++++
superset-frontend/src/views/CRUD/hooks.ts | 4 ++++
superset-frontend/src/views/CRUD/utils.tsx | 18 ++++++++++++++++++
superset/commands/chart/importers/v1/__init__.py | 9 +++++++++
superset/commands/importers/v1/utils.py | 4 ++++
superset/databases/schemas.py | 16 ++++++++++++++++
superset/models/core.py | 1 +
7 files changed, 56 insertions(+)
diff --git a/superset-frontend/src/features/databases/DatabaseModal/index.tsx
b/superset-frontend/src/features/databases/DatabaseModal/index.tsx
index 53215570a6..d596ad61cb 100644
--- a/superset-frontend/src/features/databases/DatabaseModal/index.tsx
+++ b/superset-frontend/src/features/databases/DatabaseModal/index.tsx
@@ -592,6 +592,9 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps>
= ({
const [importingModal, setImportingModal] = useState<boolean>(false);
const [importingErrorMessage, setImportingErrorMessage] = useState<string>();
const [passwordFields, setPasswordFields] = useState<string[]>([]);
+ const [encryptedExtraFields, setEncryptedExtraFields] = useState<string[]>(
+ [],
+ );
const [sshTunnelPasswordFields, setSSHTunnelPasswordFields] = useState<
string[]
>([]);
@@ -1342,6 +1345,7 @@ const DatabaseModal:
FunctionComponent<DatabaseModalProps> = ({
const onDbImport = async (info: UploadChangeParam) => {
setImportingErrorMessage('');
setPasswordFields([]);
+ setEncryptedExtraFields([]);
setSSHTunnelPasswordFields([]);
setSSHTunnelPrivateKeyFields([]);
setSSHTunnelPrivateKeyPasswordFields([]);
diff --git a/superset-frontend/src/views/CRUD/hooks.ts
b/superset-frontend/src/views/CRUD/hooks.ts
index 6fa167d934..f151e6ae16 100644
--- a/superset-frontend/src/views/CRUD/hooks.ts
+++ b/superset-frontend/src/views/CRUD/hooks.ts
@@ -30,6 +30,7 @@ import {
createErrorHandler,
getAlreadyExists,
getPasswordsNeeded,
+ getEncryptedExtraNeeded,
hasTerminalValidation,
getSSHPasswordsNeeded,
getSSHPrivateKeysNeeded,
@@ -400,6 +401,7 @@ export function useSingleViewResource<D extends object =
any>(
interface ImportResourceState {
loading: boolean;
passwordsNeeded: string[];
+ encryptedExtraNeeded: string[];
alreadyExists: string[];
sshPasswordNeeded: string[];
sshPrivateKeyNeeded: string[];
@@ -415,6 +417,7 @@ export function useImportResource(
const [state, setState] = useState<ImportResourceState>({
loading: false,
passwordsNeeded: [],
+ encryptedExtraNeeded: [],
alreadyExists: [],
sshPasswordNeeded: [],
sshPrivateKeyNeeded: [],
@@ -533,6 +536,7 @@ export function useImportResource(
} else {
updateState({
passwordsNeeded: getPasswordsNeeded(error.errors),
+ encryptedExtraNeeded: getEncryptedExtraNeeded(error.errors),
sshPasswordNeeded: getSSHPasswordsNeeded(error.errors),
sshPrivateKeyNeeded: getSSHPrivateKeysNeeded(error.errors),
sshPrivateKeyPasswordNeeded: getSSHPrivateKeyPasswordsNeeded(
diff --git a/superset-frontend/src/views/CRUD/utils.tsx
b/superset-frontend/src/views/CRUD/utils.tsx
index 7b9274a904..50b797edfb 100644
--- a/superset-frontend/src/views/CRUD/utils.tsx
+++ b/superset-frontend/src/views/CRUD/utils.tsx
@@ -395,6 +395,14 @@ const isNeedsPassword = (payload: any) =>
(e: string) => e === 'Must provide a password for the database',
);
+export /* eslint-disable no-underscore-dangle */
+const isNeedsEncryptedExtra = (payload: any) =>
+ typeof payload === 'object' &&
+ Array.isArray(payload._schema) &&
+ !!payload._schema?.find(
+ (e: string) => e === 'Must provide encrypted credentials for the database',
+ );
+
export /* eslint-disable no-underscore-dangle */
const isNeedsSSHPassword = (payload: any) =>
typeof payload === 'object' &&
@@ -433,6 +441,15 @@ export const getPasswordsNeeded = (errors: Record<string,
any>[]) =>
)
.flat();
+export const getEncryptedExtraNeeded = (errors: Record<string, any>[]) =>
+ errors
+ .map(error =>
+ Object.entries(error.extra)
+ .filter(([, payload]) => isNeedsEncryptedExtra(payload))
+ .map(([fileName]) => fileName),
+ )
+ .flat();
+
export const getSSHPasswordsNeeded = (errors: Record<string, any>[]) =>
errors
.map(error =>
@@ -482,6 +499,7 @@ export const hasTerminalValidation = (errors:
Record<string, any>[]) =>
return !noIssuesCodes.every(
([, payload]) =>
isNeedsPassword(payload) ||
+ isNeedsEncryptedExtra(payload) ||
isAlreadyExists(payload) ||
isNeedsSSHPassword(payload) ||
isNeedsSSHPrivateKey(payload) ||
diff --git a/superset/commands/chart/importers/v1/__init__.py
b/superset/commands/chart/importers/v1/__init__.py
index 89fe5e7a70..02a1b9f676 100644
--- a/superset/commands/chart/importers/v1/__init__.py
+++ b/superset/commands/chart/importers/v1/__init__.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import logging
from typing import Any
from marshmallow import Schema
@@ -31,6 +32,8 @@ from superset.daos.chart import ChartDAO
from superset.databases.schemas import ImportV1DatabaseSchema
from superset.datasets.schemas import ImportV1DatasetSchema
+logger = logging.getLogger(__name__)
+
class ImportChartsCommand(ImportModelsCommand):
"""Import charts"""
@@ -49,22 +52,28 @@ class ImportChartsCommand(ImportModelsCommand):
def _import(configs: dict[str, Any], overwrite: bool = False) -> None:
# discover datasets associated with charts
dataset_uuids: set[str] = set()
+ logger.info("parsing datasets")
for file_name, config in configs.items():
if file_name.startswith("charts/"):
dataset_uuids.add(config["dataset_uuid"])
+ logger.info("datasets parsed: %s", dataset_uuids)
# discover databases associated with datasets
+ logger.info("discovering databases associated with datasets")
+
database_uuids: set[str] = set()
for file_name, config in configs.items():
if file_name.startswith("datasets/") and config["uuid"] in
dataset_uuids:
database_uuids.add(config["database_uuid"])
# import related databases
+ logger.info("importing related dbs")
database_ids: dict[str, int] = {}
for file_name, config in configs.items():
if file_name.startswith("databases/") and config["uuid"] in
database_uuids:
database = import_database(config, overwrite=False)
database_ids[str(database.uuid)] = database.id
+ logger.info("related dbs: %s", database_ids)
# import datasets with the correct parent ref
datasets: dict[str, SqlaTable] = {}
diff --git a/superset/commands/importers/v1/utils.py
b/superset/commands/importers/v1/utils.py
index 51ab99271c..2497828f9b 100644
--- a/superset/commands/importers/v1/utils.py
+++ b/superset/commands/importers/v1/utils.py
@@ -184,6 +184,10 @@ def load_configs(
db_ssh_tunnel_priv_key_passws[config["uuid"]]
)
+ logger.info(
+ "attempting to load content into schema %s %s", schema,
content
+ )
+
schema.load(config)
configs[file_name] = config
except ValidationError as exc:
diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py
index 27eb043eb1..6d8dc4c69b 100644
--- a/superset/databases/schemas.py
+++ b/superset/databases/schemas.py
@@ -20,6 +20,7 @@
from __future__ import annotations
import inspect
+import logging
import os
from pathlib import Path
from typing import Any, TypedDict
@@ -55,6 +56,8 @@ from superset.security.analytics_db_safety import
check_sqlalchemy_uri
from superset.utils import json
from superset.utils.core import markdown, parse_ssl_cert
+logger = logging.getLogger(__name__)
+
database_schemas_query_schema = {
"type": "object",
"properties": {
@@ -859,6 +862,7 @@ class ImportV1DatabaseSchema(Schema):
allow_csv_upload = fields.Boolean()
impersonate_user = fields.Boolean()
extra = fields.Nested(ImportV1DatabaseExtraSchema)
+ encrypted_extra = fields.String(allow_none=True)
uuid = fields.UUID(required=True)
version = fields.String(required=True)
is_managed_externally = fields.Boolean(allow_none=True, dump_default=False)
@@ -878,6 +882,18 @@ class ImportV1DatabaseSchema(Schema):
if password == PASSWORD_MASK and data.get("password") is None:
raise ValidationError("Must provide a password for the database")
+ @validates_schema
+ def validate_encrypted_extra(self, data: dict[str, Any], **kwargs: Any) ->
None:
+ """If encrypted extra contains masked data, encrypted extra is
required"""
+ uuid = data["uuid"]
+ existing = db.session.query(Database).filter_by(uuid=uuid).first()
+ if existing:
+ return
+ encrypted_extra = data.get("encrypted_extra", None)
+ logger.info("encrypted_extra loaded as %s", encrypted_extra)
+ if not encrypted_extra:
+ raise ValidationError("Must provide encrypted credentials for the
database")
+
@validates_schema
def validate_ssh_tunnel_credentials(
self, data: dict[str, Any], **kwargs: Any
diff --git a/superset/models/core.py b/superset/models/core.py
index 5d3a6ea74d..5907008e0a 100755
--- a/superset/models/core.py
+++ b/superset/models/core.py
@@ -172,6 +172,7 @@ class Database(Model, AuditMixinNullable,
ImportExportMixin): # pylint: disable
"allow_file_upload",
"extra",
"impersonate_user",
+ "encrypted_extra",
]
extra_import_fields = [
"password",
