This is an automated email from the ASF dual-hosted git repository. beto pushed a commit to branch PISQL-2 in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
commit 990ca8a414d177c5c08e3e4ff57f3d4d20488133 Author: Beto Dealmeida <[email protected]> AuthorDate: Mon Dec 7 17:05:12 2020 -0800 WIP --- .../components/ImportModal/ImportModal.test.tsx | 22 +-- .../src/chart/components/ImportModal/index.tsx | 186 +++++++++++++++++++++ .../components/ImportModal/ImportModal.test.tsx | 2 +- .../src/views/CRUD/chart/ChartList.tsx | 36 +++- superset-frontend/src/views/CRUD/chart/types.ts | 27 +++ superset/charts/api.py | 8 +- superset/charts/commands/importers/dispatcher.py | 4 +- superset/charts/commands/importers/v1/__init__.py | 18 ++ superset/charts/schemas.py | 2 + superset/datasets/schemas.py | 2 +- tests/fixtures/importexport.py | 2 + 11 files changed, 293 insertions(+), 16 deletions(-) diff --git a/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx b/superset-frontend/src/chart/components/ImportModal/ImportModal.test.tsx similarity index 83% copy from superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx copy to superset-frontend/src/chart/components/ImportModal/ImportModal.test.tsx index d6e49d5..ddcf44e 100644 --- a/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx +++ b/superset-frontend/src/chart/components/ImportModal/ImportModal.test.tsx @@ -22,7 +22,7 @@ import configureStore from 'redux-mock-store'; import { styledMount as mount } from 'spec/helpers/theming'; import { ReactWrapper } from 'enzyme'; -import ImportDatasetModal from 'src/datasource/components/ImportModal'; +import ImportChartModal from 'src/chart/components/ImportModal'; import Modal from 'src/common/components/Modal'; const mockStore = configureStore([thunk]); @@ -31,16 +31,16 @@ const store = mockStore({}); const requiredProps = { addDangerToast: () => {}, addSuccessToast: () => {}, - onDatasetImport: () => {}, + onChartImport: () => {}, show: true, onHide: () => {}, }; -describe('ImportDatasetModal', () => { +describe('ImportChartModal', () => { let wrapper: ReactWrapper; beforeEach(() => { - wrapper = mount(<ImportDatasetModal {...requiredProps} />, { + wrapper = mount(<ImportChartModal {...requiredProps} />, { context: { store }, }); }); @@ -50,15 +50,15 @@ describe('ImportDatasetModal', () => { }); it('renders', () => { - expect(wrapper.find(ImportDatasetModal)).toExist(); + expect(wrapper.find(ImportChartModal)).toExist(); }); it('renders a Modal', () => { expect(wrapper.find(Modal)).toExist(); }); - it('renders "Import Dataset" header', () => { - expect(wrapper.find('h4').text()).toEqual('Import Dataset'); + it('renders "Import Chart" header', () => { + expect(wrapper.find('h4').text()).toEqual('Import Chart'); }); it('renders a label and a file input field', () => { @@ -67,7 +67,7 @@ describe('ImportDatasetModal', () => { }); it('should attach the label to the input field', () => { - const id = 'datasetFile'; + const id = 'chartFile'; expect(wrapper.find('label').prop('htmlFor')).toBe(id); expect(wrapper.find('input').prop('id')).toBe(id); }); @@ -83,7 +83,7 @@ describe('ImportDatasetModal', () => { }); it('should render the import button enabled when a file is selected', () => { - const file = new File([new ArrayBuffer(1)], 'dataset_export.zip'); + const file = new File([new ArrayBuffer(1)], 'chart_export.zip'); wrapper.find('input').simulate('change', { target: { files: [file] } }); expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe( @@ -93,9 +93,9 @@ describe('ImportDatasetModal', () => { it('should render password fields when needed for import', () => { const wrapperWithPasswords = mount( - <ImportDatasetModal + <ImportChartModal {...requiredProps} - passwordFields={['datasets/examples.yaml']} + passwordFields={['databases/examples.yaml']} />, { context: { store }, diff --git a/superset-frontend/src/chart/components/ImportModal/index.tsx b/superset-frontend/src/chart/components/ImportModal/index.tsx new file mode 100644 index 0000000..328af18 --- /dev/null +++ b/superset-frontend/src/chart/components/ImportModal/index.tsx @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React, { FunctionComponent, useEffect, useRef, useState } from 'react'; +import { t } from '@superset-ui/core'; + +import Modal from 'src/common/components/Modal'; +import { + StyledIcon, + StyledInputContainer, +} from 'src/views/CRUD/data/database/DatabaseModal'; +import { useImportResource } from 'src/views/CRUD/hooks'; +import { ChartObject } from 'src/views/CRUD/chart/types'; + +export interface ImportChartModalProps { + addDangerToast: (msg: string) => void; + addSuccessToast: (msg: string) => void; + onChartImport: () => void; + show: boolean; + onHide: () => void; + passwordFields?: string[]; + setPasswordFields?: (passwordFields: string[]) => void; +} + +const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({ + addDangerToast, + addSuccessToast, + onChartImport, + show, + onHide, + passwordFields = [], + setPasswordFields = () => {}, +}) => { + const [uploadFile, setUploadFile] = useState<File | null>(null); + const [isHidden, setIsHidden] = useState<boolean>(true); + const [passwords, setPasswords] = useState<Record<string, string>>({}); + const fileInputRef = useRef<HTMLInputElement>(null); + + const clearModal = () => { + setUploadFile(null); + setPasswordFields([]); + setPasswords({}); + if (fileInputRef && fileInputRef.current) { + fileInputRef.current.value = ''; + } + }; + + const handleErrorMsg = (msg: string) => { + clearModal(); + addDangerToast(msg); + }; + + const { + state: { passwordsNeeded }, + importResource, + } = useImportResource<ChartObject>('chart', t('chart'), handleErrorMsg); + + useEffect(() => { + setPasswordFields(passwordsNeeded); + }, [passwordsNeeded]); + + // Functions + const hide = () => { + setIsHidden(true); + onHide(); + }; + + const onUpload = () => { + if (uploadFile === null) { + return; + } + + importResource(uploadFile, passwords).then(result => { + if (result) { + addSuccessToast(t('The charts have been imported')); + clearModal(); + onChartImport(); + } + }); + }; + + const changeFile = (event: React.ChangeEvent<HTMLInputElement>) => { + const { files } = event.target as HTMLInputElement; + setUploadFile((files && files[0]) || null); + }; + + const renderPasswordFields = () => { + if (passwordFields.length === 0) { + return null; + } + + return ( + <> + <h5>Database passwords</h5> + <StyledInputContainer> + <div className="helper"> + {t( + 'The passwords for the databases below are needed in order to ' + + 'import them together with the charts. Please note that the ' + + '"Secure Extra" and "Certificate" sections of ' + + 'the database configuration are not present in export files, and ' + + 'should be added manually after the import if they are needed.', + )} + </div> + </StyledInputContainer> + {passwordFields.map(fileName => ( + <StyledInputContainer key={`password-for-${fileName}`}> + <div className="control-label"> + {fileName} + <span className="required">*</span> + </div> + <input + name={`password-${fileName}`} + autoComplete="off" + type="password" + value={passwords[fileName]} + onChange={event => + setPasswords({ ...passwords, [fileName]: event.target.value }) + } + /> + </StyledInputContainer> + ))} + </> + ); + }; + + // Show/hide + if (isHidden && show) { + setIsHidden(false); + } + + return ( + <Modal + name="chart" + className="chart-modal" + disablePrimaryButton={uploadFile === null} + onHandledPrimaryAction={onUpload} + onHide={hide} + primaryButtonName={t('Import')} + width="750px" + show={show} + title={ + <h4> + <StyledIcon name="nav-charts" /> + {t('Import Chart')} + </h4> + } + > + <StyledInputContainer> + <div className="control-label"> + <label htmlFor="chartFile"> + {t('File')} + <span className="required">*</span> + </label> + </div> + <input + ref={fileInputRef} + data-test="chart-file-input" + name="chartFile" + id="chartFile" + type="file" + accept=".yaml,.json,.yml,.zip" + onChange={changeFile} + /> + </StyledInputContainer> + {renderPasswordFields()} + </Modal> + ); +}; + +export default ImportChartModal; diff --git a/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx b/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx index d6e49d5..7087129 100644 --- a/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx +++ b/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx @@ -95,7 +95,7 @@ describe('ImportDatasetModal', () => { const wrapperWithPasswords = mount( <ImportDatasetModal {...requiredProps} - passwordFields={['datasets/examples.yaml']} + passwordFields={['databases/examples.yaml']} />, { context: { store }, diff --git a/superset-frontend/src/views/CRUD/chart/ChartList.tsx b/superset-frontend/src/views/CRUD/chart/ChartList.tsx index e7cf14a..ea787e1 100644 --- a/superset-frontend/src/views/CRUD/chart/ChartList.tsx +++ b/superset-frontend/src/views/CRUD/chart/ChartList.tsx @@ -17,7 +17,7 @@ * under the License. */ import { SupersetClient, getChartMetadataRegistry, t } from '@superset-ui/core'; -import React, { useMemo } from 'react'; +import React, { useMemo, useState } from 'react'; import rison from 'rison'; import { uniqBy } from 'lodash'; import { isFeatureEnabled, FeatureFlag } from 'src/featureFlags'; @@ -43,6 +43,7 @@ import ListView, { } from 'src/components/ListView'; import withToasts from 'src/messageToasts/enhancers/withToasts'; import PropertiesModal from 'src/explore/components/PropertiesModal'; +import ImportChartModal from 'src/chart/components/ImportModal/index'; import Chart from 'src/types/Chart'; import TooltipWrapper from 'src/components/TooltipWrapper'; import ChartCard from './ChartCard'; @@ -124,6 +125,22 @@ function ChartList(props: ChartListProps) { closeChartEditModal, } = useChartEditModal(setCharts, charts); + const [importingChart, showImportModal] = useState<boolean>(false); + const [passwordFields, setPasswordFields] = useState<string[]>([]); + + function openChartImportModal() { + showImportModal(true); + } + + function closeChartImportModal() { + showImportModal(false); + } + + const handleChartImport = () => { + showImportModal(false); + refreshData(); + }; + const canCreate = hasPerm('can_add'); const canEdit = hasPerm('can_edit'); const canDelete = hasPerm('can_delete'); @@ -482,6 +499,13 @@ function ChartList(props: ChartListProps) { }, }); } + if (isFeatureEnabled(FeatureFlag.VERSIONED_EXPORT)) { + subMenuButtons.push({ + name: <Icon name="import" />, + buttonStyle: 'link', + onClick: openChartImportModal, + }); + } return ( <> <SubMenu name={t('Charts')} buttons={subMenuButtons} /> @@ -541,6 +565,16 @@ function ChartList(props: ChartListProps) { ); }} </ConfirmStatusChange> + + <ImportChartModal + show={importingChart} + onHide={closeChartImportModal} + addDangerToast={props.addDangerToast} + addSuccessToast={props.addSuccessToast} + onChartImport={handleChartImport} + passwordFields={passwordFields} + setPasswordFields={setPasswordFields} + /> </> ); } diff --git a/superset-frontend/src/views/CRUD/chart/types.ts b/superset-frontend/src/views/CRUD/chart/types.ts new file mode 100644 index 0000000..209d009 --- /dev/null +++ b/superset-frontend/src/views/CRUD/chart/types.ts @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export type ChartObject = { + slice_name?: string; + description?: string; + viz_type?: string; + params?: string; + cache_timeout?: number; + datasource_id?: number; + datasource_type?: number; +}; diff --git a/superset/charts/api.py b/superset/charts/api.py index 262d2cf..59a8dfc 100644 --- a/superset/charts/api.py +++ b/superset/charts/api.py @@ -878,7 +878,13 @@ class ChartRestApi(BaseSupersetModelRestApi): for file_name in bundle.namelist() } - command = ImportChartsCommand(contents) + passwords = ( + json.loads(request.form["passwords"]) + if "passwords" in request.form + else None + ) + + command = ImportChartsCommand(contents, passwords=passwords) try: command.run() return self.response(200, message="OK") diff --git a/superset/charts/commands/importers/dispatcher.py b/superset/charts/commands/importers/dispatcher.py index 2098ea7..ff34889 100644 --- a/superset/charts/commands/importers/dispatcher.py +++ b/superset/charts/commands/importers/dispatcher.py @@ -43,12 +43,14 @@ class ImportChartsCommand(BaseCommand): # pylint: disable=unused-argument def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any): self.contents = contents + self.args = args + self.kwargs = kwargs def run(self) -> None: # iterate over all commands until we find a version that can # handle the contents for version in command_versions: - command = version(self.contents) + command = version(self.contents, *self.args, **self.kwargs) try: command.run() return diff --git a/superset/charts/commands/importers/v1/__init__.py b/superset/charts/commands/importers/v1/__init__.py index 4aed3fa..69840f6 100644 --- a/superset/charts/commands/importers/v1/__init__.py +++ b/superset/charts/commands/importers/v1/__init__.py @@ -36,6 +36,7 @@ from superset.databases.commands.importers.v1.utils import import_database from superset.databases.schemas import ImportV1DatabaseSchema from superset.datasets.commands.importers.v1.utils import import_dataset from superset.datasets.schemas import ImportV1DatasetSchema +from superset.models.core import Database from superset.models.slice import Slice schemas: Dict[str, Schema] = { @@ -52,6 +53,7 @@ class ImportChartsCommand(BaseCommand): # pylint: disable=unused-argument def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any): self.contents = contents + self.passwords: Dict[str, str] = kwargs.get("passwords") or {} self._configs: Dict[str, Any] = {} def _import_bundle(self, session: Session) -> None: @@ -113,6 +115,14 @@ class ImportChartsCommand(BaseCommand): def validate(self) -> None: exceptions: List[ValidationError] = [] + # load existing databases so we can apply the password validation + db_passwords = { + str(uuid): password + for uuid, password in db.session.query( + Database.uuid, Database.password + ).all() + } + # verify that the metadata file is present and valid try: metadata: Optional[Dict[str, str]] = load_metadata(self.contents) @@ -120,12 +130,20 @@ class ImportChartsCommand(BaseCommand): exceptions.append(exc) metadata = None + # validate charts, datasets, and databases for file_name, content in self.contents.items(): prefix = file_name.split("/")[0] schema = schemas.get(f"{prefix}/") if schema: try: config = load_yaml(file_name, content) + + # populate passwords from the request or from existing DBs + if file_name in self.passwords: + config["password"] = self.passwords[file_name] + elif prefix == "databases" and config["uuid"] in db_passwords: + config["password"] = db_passwords[config["uuid"]] + schema.load(config) self._configs[file_name] = config except ValidationError as exc: diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py index 5e346fa..347189a 100644 --- a/superset/charts/schemas.py +++ b/superset/charts/schemas.py @@ -1118,6 +1118,8 @@ class GetFavStarIdsSchema(Schema): class ImportV1ChartSchema(Schema): + slice_name = fields.String(required=True) + viz_type = fields.String(required=True) params = fields.Dict() cache_timeout = fields.Integer(allow_none=True) uuid = fields.UUID(required=True) diff --git a/superset/datasets/schemas.py b/superset/datasets/schemas.py index 373e49f..b8406eb 100644 --- a/superset/datasets/schemas.py +++ b/superset/datasets/schemas.py @@ -129,7 +129,7 @@ class ImportV1ColumnSchema(Schema): verbose_name = fields.String(allow_none=True) is_dttm = fields.Boolean() is_active = fields.Boolean(allow_none=True) - type = fields.String(required=True) + type = fields.String(allow_none=True) groupby = fields.Boolean() filterable = fields.Boolean() expression = fields.String(allow_none=True) diff --git a/tests/fixtures/importexport.py b/tests/fixtures/importexport.py index 6383923..f0c35a1 100644 --- a/tests/fixtures/importexport.py +++ b/tests/fixtures/importexport.py @@ -404,6 +404,8 @@ dataset_config: Dict[str, Any] = { } chart_config: Dict[str, Any] = { + "slice_name": "Deck Path", + "viz_type": "deck_path", "params": { "color_picker": {"a": 1, "b": 135, "g": 122, "r": 0}, "datasource": "12__table",
