IGNITE-5466 Web Console: Configuration reworked to cluster centric model: 1. Reworked data model. 2. Implemented migrations. 3. Reworked UI for all screens. 4. Reworked validation. 5. Many refactorings to improve code base. 6. Added tests. 7. Many minor improvements.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7ee1683e Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7ee1683e Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7ee1683e Branch: refs/heads/master Commit: 7ee1683e1677c05291a8de77005ce3c1ff5b9890 Parents: f2d800e Author: Ilya Borisov <[email protected]> Authored: Mon Apr 2 19:24:23 2018 +0700 Committer: Alexey Kuznetsov <[email protected]> Committed: Mon Apr 2 19:24:30 2018 +0700 ---------------------------------------------------------------------- modules/web-console/backend/app/apiServer.js | 9 +- modules/web-console/backend/app/routes.js | 5 +- modules/web-console/backend/app/schemas.js | 10 +- .../backend/errors/AppErrorException.js | 2 - .../backend/errors/AuthFailedException.js | 2 +- .../backend/errors/IllegalAccessError.js | 3 +- .../backend/errors/IllegalArgumentException.js | 1 - .../backend/errors/MissingResourceException.js | 2 +- .../backend/errors/ServerErrorException.js | 1 - modules/web-console/backend/middlewares/api.js | 15 +- modules/web-console/backend/middlewares/demo.js | 31 + .../1502249492000-invalidate_rename.js | 28 + .../migrations/1502432624000-cache-index.js | 32 + .../migrations/1504672035000-igfs-index.js | 32 + .../migrations/1505114649000-models-index.js | 32 + .../1508395969410-init-registered-date.js | 7 +- .../migrations/1516948939797-migrate-configs.js | 346 + .../backend/migrations/migration-utils.js | 153 + .../backend/migrations/recreate-index.js | 30 - modules/web-console/backend/routes/caches.js | 12 + modules/web-console/backend/routes/clusters.js | 46 +- .../web-console/backend/routes/configuration.js | 12 +- modules/web-console/backend/routes/domains.js | 6 + modules/web-console/backend/routes/igfss.js | 12 + modules/web-console/backend/services/caches.js | 78 +- .../web-console/backend/services/clusters.js | 153 +- .../backend/services/configurations.js | 12 + modules/web-console/backend/services/domains.js | 96 +- modules/web-console/backend/services/igfss.js | 44 +- .../web-console/backend/services/sessions.js | 2 +- modules/web-console/backend/services/spaces.js | 2 +- .../backend/test/unit/CacheService.test.js | 45 +- .../backend/test/unit/ClusterService.test.js | 233 +- .../backend/test/unit/DomainService.test.js | 5 + .../e2e/testcafe/components/ListEditable.js | 83 + .../e2e/testcafe/components/Table.js | 56 + .../components/pageAdvancedConfiguration.js | 39 + .../testcafe/components/pageConfiguration.js | 21 + .../testcafe/fixtures/configuration/basic.js | 89 + .../testcafe/fixtures/configuration/overview.js | 147 + .../e2e/testcafe/fixtures/menu-smoke.js | 2 +- modules/web-console/e2e/testcafe/package.json | 2 +- .../PageConfigurationAdvancedCluster.js | 28 + .../page-models/PageConfigurationBasic.js | 68 + .../page-models/PageConfigurationOverview.js | 36 + .../pageConfigurationAdvancedIGFS.js | 21 + .../pageConfigurationAdvancedModels.js | 28 + modules/web-console/e2e/testcafe/roles.js | 1 - modules/web-console/frontend/.babelrc | 2 +- modules/web-console/frontend/.eslintrc | 2 +- modules/web-console/frontend/.gitignore | 5 + modules/web-console/frontend/app/app.config.js | 29 +- modules/web-console/frontend/app/app.d.ts | 29 + modules/web-console/frontend/app/app.js | 26 +- .../app/components/bs-select-menu/style.scss | 4 +- .../directives.js | 53 + .../expose-ignite-form-field-control/index.js | 23 + .../grid-column-selector/template.pug | 1 + .../components/grid-item-selected/controller.js | 2 +- .../app/components/ignite-icon/directive.js | 2 +- .../app/components/ignite-icon/style.scss | 8 +- .../list-editable-add-item-button/component.js | 86 + .../component.spec.js | 72 + .../has-items-template.pug | 23 + .../list-editable-add-item-button/index.js | 24 + .../no-items-template.pug | 18 + .../list-editable-add-item-button/style.scss | 21 + .../list-editable-cols/cols.directive.js | 5 +- .../list-editable-cols/cols.style.scss | 16 +- .../list-editable-cols/cols.template.pug | 2 +- .../components/list-editable-cols/index.js | 3 +- .../list-editable-cols/row.directive.js | 4 +- .../list-editable-one-way/directive.js | 54 + .../components/list-editable-one-way/index.js | 24 + .../list-editable-save-on-changes/directives.js | 76 + .../list-editable-save-on-changes/index.js | 24 + .../list-editable-transclude/directive.js | 3 + .../app/components/list-editable/controller.js | 59 +- .../app/components/list-editable/index.js | 8 +- .../app/components/list-editable/style.scss | 31 +- .../app/components/list-editable/template.pug | 9 +- .../list-of-registered-users/column-defs.js | 16 +- .../list-of-registered-users/controller.js | 14 +- .../components/cache-edit-form/component.js | 32 + .../components/cache-edit-form/controller.js | 104 + .../components/cache-edit-form/index.js | 21 + .../components/cache-edit-form/style.scss | 20 + .../components/cache-edit-form/template.tpl.pug | 48 + .../components/cluster-edit-form/component.js | 31 + .../components/cluster-edit-form/controller.js | 118 + .../components/cluster-edit-form/index.js | 21 + .../components/cluster-edit-form/style.scss | 20 + .../cluster-edit-form/template.tpl.pug | 88 + .../components/igfs-edit-form/component.js | 30 + .../components/igfs-edit-form/controller.js | 60 + .../components/igfs-edit-form/index.js | 21 + .../components/igfs-edit-form/style.scss | 20 + .../components/igfs-edit-form/template.tpl.pug | 39 + .../components/model-edit-form/component.js | 31 + .../components/model-edit-form/controller.js | 190 + .../components/model-edit-form/index.js | 21 + .../components/model-edit-form/style.scss | 20 + .../components/model-edit-form/template.tpl.pug | 33 + .../page-configure-advanced-caches/component.js | 25 + .../controller.js | 174 + .../page-configure-advanced-caches/index.js | 23 + .../page-configure-advanced-caches/template.pug | 57 + .../component.js | 25 + .../controller.js | 51 + .../page-configure-advanced-cluster/index.js | 23 + .../template.pug | 25 + .../page-configure-advanced-igfs/component.js | 25 + .../page-configure-advanced-igfs/controller.js | 139 + .../page-configure-advanced-igfs/index.js | 23 + .../page-configure-advanced-igfs/template.pug | 51 + .../page-configure-advanced-models/component.js | 26 + .../controller.js | 171 + .../hasIndex.template.pug | 23 + .../page-configure-advanced-models/index.js | 23 + .../keyCell.template.pug | 21 + .../page-configure-advanced-models/style.scss | 37 + .../page-configure-advanced-models/template.pug | 51 + .../valueCell.template.pug | 18 + .../page-configure-advanced/controller.js | 15 +- .../components/page-configure-advanced/index.js | 23 +- .../page-configure-advanced/service.js | 31 - .../page-configure-advanced/style.scss | 139 +- .../page-configure-advanced/template.pug | 14 +- .../components/pcbScaleNumber.js | 46 - .../page-configure-basic/controller.js | 242 +- .../page-configure-basic/controller.spec.js | 19 +- .../components/page-configure-basic/index.js | 11 +- .../mixins/pcb-form-field-size.pug | 71 - .../components/page-configure-basic/reducer.js | 17 +- .../page-configure-basic/reducer.spec.js | 2 +- .../components/page-configure-basic/service.js | 134 - .../page-configure-basic/service.spec.js | 323 - .../components/page-configure-basic/style.scss | 131 +- .../page-configure-basic/template.pug | 281 +- .../page-configure-overview/component.js | 25 + .../pco-grid-column-categories/directive.js | 67 + .../page-configure-overview/controller.js | 163 + .../components/page-configure-overview/index.js | 26 + .../page-configure-overview/style.scss | 33 + .../page-configure-overview/template.pug | 40 + .../app/components/page-configure/component.js | 5 +- .../button-download-project/component.js | 36 + .../components/button-download-project/index.js | 23 + .../button-download-project/template.pug | 22 + .../button-import-models/component.js | 37 + .../components/button-import-models/index.js | 23 + .../components/button-import-models/style.scss | 25 + .../button-import-models/template.pug | 20 + .../button-preview-project/component.js | 36 + .../components/button-preview-project/index.js | 23 + .../button-preview-project/template.pug | 22 + .../page-configure/components/fakeUICanExit.js | 48 + .../components/formUICanExitGuard.js | 59 + .../components/modal-import-models/component.js | 1151 ++ .../components/modal-import-models/index.js | 31 + .../component.js | 27 + .../selected-items-amount-indicator/style.scss | 24 + .../template.pug | 17 + .../components/modal-import-models/service.js | 56 + .../step-indicator/component.js | 35 + .../step-indicator/style.scss | 101 + .../step-indicator/template.pug | 31 + .../components/modal-import-models/style.scss | 53 + .../tables-action-cell/component.js | 62 + .../tables-action-cell/style.scss | 49 + .../tables-action-cell/template.pug | 45 + .../modal-import-models/template.tpl.pug | 181 + .../modal-preview-project/component.js | 31 + .../modal-preview-project/controller.js | 120 + .../components/modal-preview-project/index.js | 27 + .../components/modal-preview-project/service.js | 52 + .../components/modal-preview-project/style.scss | 67 + .../modal-preview-project/template.pug | 47 + .../components/pc-form-field-size/component.js | 41 + .../components/pc-form-field-size/controller.js | 131 + .../components/pc-form-field-size/index.js | 23 + .../components/pc-form-field-size/style.scss | 52 + .../components/pc-form-field-size/template.pug | 61 + .../components/pc-items-table/component.js | 45 + .../components/pc-items-table/controller.js | 125 + .../components/pc-items-table/decorator.js | 34 + .../components/pc-items-table/index.js | 25 + .../components/pc-items-table/style.scss | 71 + .../components/pc-items-table/template.pug | 49 + .../components/pc-ui-grid-filters/directive.js | 62 + .../components/pc-ui-grid-filters/index.js | 43 + .../components/pc-ui-grid-filters/style.scss | 22 + .../components/pc-ui-grid-filters/template.pug | 39 + .../components/pcIsInCollection.js | 41 + .../page-configure/components/pcValidation.js | 192 + .../app/components/page-configure/controller.js | 35 +- .../components/page-configure/defaultNames.js | 23 + .../app/components/page-configure/index.d.ts | 151 + .../app/components/page-configure/index.js | 136 +- .../app/components/page-configure/reducer.js | 353 +- .../components/page-configure/reducer.spec.js | 21 +- .../page-configure/reduxDevtoolsIntegration.js | 75 + .../services/ConfigChangesGuard.js | 66 + .../services/ConfigSelectionManager.js | 93 + .../services/ConfigurationDownload.js | 23 +- .../services/ConfigurationDownload.spec.js | 2 +- .../page-configure/services/ConfigureState.js | 90 +- .../page-configure/services/PageConfigure.js | 86 +- .../services/PageConfigure.spec.js | 244 + .../page-configure/store/actionCreators.js | 170 + .../page-configure/store/actionTypes.js | 31 + .../components/page-configure/store/effects.js | 664 + .../page-configure/store/selectors.js | 170 + .../app/components/page-configure/style.scss | 285 +- .../app/components/page-configure/template.pug | 43 +- .../transitionHooks/errorState.js | 55 + .../page-configure/types/uirouter.d.ts | 20 + .../app/components/page-profile/style.scss | 4 + .../components/queries-notebook/controller.js | 24 +- .../queries-notebooks-list/controller.js | 8 +- .../queries-notebooks-list/template.tpl.pug | 2 +- .../app/components/version-picker/style.scss | 3 +- .../app/components/version-picker/template.pug | 14 +- .../app/core/activities/Activities.data.d.ts | 37 + .../app/core/activities/Activities.data.js | 10 +- .../frontend/app/data/getting-started.json | 17 - modules/web-console/frontend/app/data/i18n.js | 18 +- .../app/directives/on-focus-out.directive.js | 100 +- .../ui-ace-pojos/ui-ace-pojos.controller.js | 4 +- .../app/directives/ui-ace.controller.js | 35 +- .../frontend/app/helpers/jade/form.pug | 2 - .../helpers/jade/form/form-field-checkbox.pug | 43 +- .../helpers/jade/form/form-field-datalist.pug | 25 +- .../helpers/jade/form/form-field-dropdown.pug | 34 +- .../helpers/jade/form/form-field-feedback.pug | 16 +- .../app/helpers/jade/form/form-field-label.pug | 8 +- .../app/helpers/jade/form/form-field-number.pug | 41 +- .../helpers/jade/form/form-field-password.pug | 26 +- .../app/helpers/jade/form/form-field-text.pug | 28 +- .../app/helpers/jade/form/form-group.pug | 23 - .../frontend/app/helpers/jade/mixins.pug | 400 +- .../app/modules/agent/AgentManager.service.js | 9 +- .../modules/configuration/generator/Beans.js | 60 +- .../generator/ConfigurationGenerator.js | 43 +- .../generator/JavaTransformer.service.js | 24 +- .../generator/PlatformGenerator.js | 8 +- .../generator/SpringTransformer.service.js | 2 +- .../frontend/app/modules/demo/Demo.module.js | 6 +- .../field/bs-select-placeholder.directive.js | 20 +- .../frontend/app/modules/form/form.module.js | 6 - .../app/modules/form/panel/chevron.directive.js | 17 +- .../app/modules/form/panel/field.directive.js | 69 - .../app/modules/form/panel/panel.directive.js | 37 - .../app/modules/form/panel/revert.directive.js | 54 - .../form/validator/java-identifier.directive.js | 5 +- .../modules/form/validator/unique.directive.js | 78 +- .../modules/nodes/nodes-dialog.controller.js | 2 +- .../app/modules/states/configuration.state.js | 283 +- .../states/configuration/caches/affinity.pug | 78 +- .../configuration/caches/client-near-cache.pug | 50 - .../states/configuration/caches/concurrency.pug | 29 +- .../states/configuration/caches/general.pug | 81 +- .../states/configuration/caches/memory.pug | 219 +- .../configuration/caches/near-cache-client.pug | 44 +- .../configuration/caches/near-cache-server.pug | 46 +- .../states/configuration/caches/node-filter.pug | 51 +- .../states/configuration/caches/query.pug | 132 +- .../states/configuration/caches/rebalance.pug | 63 +- .../states/configuration/caches/statistics.pug | 25 +- .../states/configuration/caches/store.pug | 450 +- .../states/configuration/clusters/atomic.pug | 61 +- .../configuration/clusters/attributes.pug | 57 +- .../states/configuration/clusters/binary.pug | 102 +- .../configuration/clusters/cache-key-cfg.pug | 72 +- .../configuration/clusters/checkpoint.pug | 119 +- .../configuration/clusters/checkpoint/fs.pug | 66 +- .../configuration/clusters/checkpoint/jdbc.pug | 51 +- .../configuration/clusters/checkpoint/s3.pug | 138 +- .../configuration/clusters/client-connector.pug | 83 +- .../states/configuration/clusters/collision.pug | 46 +- .../configuration/clusters/collision/custom.pug | 9 +- .../clusters/collision/fifo-queue.pug | 15 +- .../clusters/collision/job-stealing.pug | 74 +- .../clusters/collision/priority-queue.pug | 45 +- .../configuration/clusters/communication.pug | 139 +- .../states/configuration/clusters/connector.pug | 63 +- .../configuration/clusters/data-storage.pug | 380 +- .../configuration/clusters/deployment.pug | 290 +- .../states/configuration/clusters/discovery.pug | 92 +- .../states/configuration/clusters/events.pug | 51 +- .../states/configuration/clusters/failover.pug | 122 +- .../states/configuration/clusters/general.pug | 109 +- .../clusters/general/discovery/cloud.pug | 144 +- .../clusters/general/discovery/google.pug | 12 +- .../clusters/general/discovery/jdbc.pug | 25 +- .../clusters/general/discovery/kubernetes.pug | 12 +- .../clusters/general/discovery/multicast.pug | 101 +- .../clusters/general/discovery/s3.pug | 28 +- .../clusters/general/discovery/shared.pug | 8 +- .../clusters/general/discovery/vm.pug | 95 +- .../clusters/general/discovery/zookeeper.pug | 93 +- .../retrypolicy/bounded-exponential-backoff.pug | 13 +- .../discovery/zookeeper/retrypolicy/custom.pug | 3 +- .../retrypolicy/exponential-backoff.pug | 13 +- .../discovery/zookeeper/retrypolicy/forever.pug | 3 +- .../discovery/zookeeper/retrypolicy/n-times.pug | 9 +- .../zookeeper/retrypolicy/one-time.pug | 6 +- .../zookeeper/retrypolicy/until-elapsed.pug | 9 +- .../states/configuration/clusters/hadoop.pug | 120 +- .../states/configuration/clusters/igfs.pug | 25 +- .../configuration/clusters/load-balancing.pug | 177 +- .../states/configuration/clusters/logger.pug | 40 +- .../configuration/clusters/logger/custom.pug | 9 +- .../configuration/clusters/logger/log4j.pug | 59 +- .../configuration/clusters/logger/log4j2.pug | 37 +- .../configuration/clusters/marshaller.pug | 106 +- .../states/configuration/clusters/memory.pug | 262 +- .../states/configuration/clusters/metrics.pug | 29 +- .../states/configuration/clusters/misc.pug | 60 +- .../states/configuration/clusters/odbc.pug | 36 +- .../configuration/clusters/persistence.pug | 61 +- .../states/configuration/clusters/service.pug | 132 +- .../configuration/clusters/sql-connector.pug | 45 +- .../states/configuration/clusters/ssl.pug | 149 +- .../states/configuration/clusters/swap.pug | 83 +- .../states/configuration/clusters/thread.pug | 160 +- .../states/configuration/clusters/time.pug | 42 +- .../configuration/clusters/transactions.pug | 35 +- .../states/configuration/domains/general.pug | 48 +- .../states/configuration/domains/query.pug | 392 +- .../states/configuration/domains/store.pug | 191 +- .../modules/states/configuration/igfs/dual.pug | 21 +- .../states/configuration/igfs/fragmentizer.pug | 25 +- .../states/configuration/igfs/general.pug | 86 +- .../modules/states/configuration/igfs/ipc.pug | 34 +- .../modules/states/configuration/igfs/misc.pug | 136 +- .../states/configuration/igfs/secondary.pug | 45 +- .../summary/summary-tabs.directive.js | 50 - .../summary/summary-zipper.service.js | 2 + .../configuration/summary/summary.controller.js | 350 - .../configuration/summary/summary.worker.js | 17 +- .../frontend/app/primitives/btn/index.scss | 21 + .../frontend/app/primitives/checkbox/index.scss | 52 + .../app/primitives/datepicker/index.pug | 8 +- .../frontend/app/primitives/dropdown/index.pug | 6 +- .../frontend/app/primitives/file/index.pug | 2 +- .../app/primitives/form-field/index.scss | 96 +- .../frontend/app/primitives/index.js | 1 + .../frontend/app/primitives/modal/index.scss | 1 + .../frontend/app/primitives/radio/index.pug | 12 +- .../frontend/app/primitives/tabs/index.scss | 10 +- .../app/primitives/timepicker/index.pug | 8 +- .../frontend/app/primitives/tooltip/index.pug | 3 +- .../frontend/app/primitives/ui-grid/index.scss | 11 +- .../web-console/frontend/app/services/Caches.js | 206 +- .../frontend/app/services/Clusters.js | 483 +- .../frontend/app/services/Confirm.service.js | 38 + .../app/services/ConfirmBatch.service.js | 125 +- .../app/services/ErrorPopover.service.js | 12 +- .../frontend/app/services/FormUtils.service.js | 21 +- .../web-console/frontend/app/services/IGFSs.js | 77 + .../frontend/app/services/JavaTypes.service.js | 27 +- .../app/services/LegacyUtils.service.js | 2 + .../frontend/app/services/Messages.service.js | 6 +- .../web-console/frontend/app/services/Models.js | 181 + .../frontend/app/services/Version.service.js | 1 + .../web-console/frontend/app/services/index.js | 2 + .../frontend/app/utils/lodashMixins.js | 23 + .../frontend/app/utils/uniqueName.js | 27 + modules/web-console/frontend/app/vendor.js | 1 + .../frontend/controllers/caches-controller.js | 653 - .../frontend/controllers/clusters-controller.js | 1044 -- .../frontend/controllers/domains-controller.js | 1897 --- .../frontend/controllers/igfs-controller.js | 415 - modules/web-console/frontend/package-lock.json | 11965 +++++++++++++++++ modules/web-console/frontend/package.json | 25 +- .../frontend/public/images/checkbox-active.svg | 2 +- .../frontend/public/images/collapse.svg | 3 + .../frontend/public/images/expand.svg | 3 + .../frontend/public/images/icons/collapse.svg | 2 +- .../frontend/public/images/icons/expand.svg | 2 +- .../frontend/public/images/icons/index.js | 8 +- .../frontend/public/images/icons/plus.svg | 5 +- .../frontend/public/images/icons/structure.svg | 3 + .../frontend/public/stylesheets/style.scss | 120 +- modules/web-console/frontend/tsconfig.json | 6 +- modules/web-console/frontend/views/base2.pug | 4 +- .../frontend/views/configuration/caches.tpl.pug | 55 - .../views/configuration/clusters.tpl.pug | 95 - .../views/configuration/domains.tpl.pug | 89 +- .../frontend/views/configuration/igfs.tpl.pug | 54 - .../summary-project-structure.tpl.pug | 28 - .../views/configuration/summary-tabs.pug | 25 - .../views/configuration/summary.tpl.pug | 87 - .../frontend/views/includes/header-left.pug | 8 +- .../views/templates/batch-confirm.tpl.pug | 29 +- .../frontend/views/templates/confirm.tpl.pug | 2 +- .../frontend/webpack/webpack.common.js | 4 +- .../frontend/webpack/webpack.dev.babel.js | 1 + .../frontend/webpack/webpack.test.js | 9 +- 400 files changed, 29558 insertions(+), 10491 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/app/apiServer.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/app/apiServer.js b/modules/web-console/backend/app/apiServer.js index 0030529..90c39ba 100644 --- a/modules/web-console/backend/app/apiServer.js +++ b/modules/web-console/backend/app/apiServer.js @@ -60,12 +60,11 @@ module.exports = { } // Catch 404 and forward to error handler. - app.use((req, res, next) => { - const err = new Error('Not Found: ' + req.originalUrl); + app.use((req, res) => { + if (req.xhr) + return res.status(404).send({ error: 'Not Found: ' + req.originalUrl }); - err.status = 404; - - next(err); + return res.sendStatus(404); }); // Production error handler: no stacktraces leaked to user. http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/app/routes.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/app/routes.js b/modules/web-console/backend/app/routes.js index aa6efae..ce7b5d8 100644 --- a/modules/web-console/backend/app/routes.js +++ b/modules/web-console/backend/app/routes.js @@ -43,7 +43,8 @@ module.exports.factory = function(publicRoute, adminRoute, profilesRoute, demoRo res.status(401).send('Access denied. You are not authorized to access this page.'); }; - // Registering the standard routes + // Registering the standard routes. + // NOTE: Order is important! app.use('/api/v1/', publicRoute); app.use('/api/v1/admin', _mustAuthenticated, _adminOnly, adminRoute); app.use('/api/v1/profile', _mustAuthenticated, profilesRoute); @@ -51,11 +52,11 @@ module.exports.factory = function(publicRoute, adminRoute, profilesRoute, demoRo app.all('/api/v1/configuration/*', _mustAuthenticated); - app.use('/api/v1/configuration', configurationsRoute); app.use('/api/v1/configuration/clusters', clustersRoute); app.use('/api/v1/configuration/domains', domainsRoute); app.use('/api/v1/configuration/caches', cachesRoute); app.use('/api/v1/configuration/igfs', igfssRoute); + app.use('/api/v1/configuration', configurationsRoute); app.use('/api/v1/notebooks', _mustAuthenticated, notebooksRoute); app.use('/api/v1/downloads', _mustAuthenticated, downloadsRoute); http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/app/schemas.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/app/schemas.js b/modules/web-console/backend/app/schemas.js index 553d2b2..3f37487 100644 --- a/modules/web-console/backend/app/schemas.js +++ b/modules/web-console/backend/app/schemas.js @@ -92,6 +92,7 @@ module.exports.factory = function(mongoose) { // Define Domain model schema. const DomainModel = new Schema({ space: {type: ObjectId, ref: 'Space', index: true, required: true}, + clusters: [{type: ObjectId, ref: 'Cluster'}], caches: [{type: ObjectId, ref: 'Cache'}], queryMetadata: {type: String, enum: ['Annotations', 'Configuration']}, kind: {type: String, enum: ['query', 'store', 'both']}, @@ -125,7 +126,7 @@ module.exports.factory = function(mongoose) { generatePojo: Boolean }); - DomainModel.index({valueType: 1, space: 1}, {unique: true}); + DomainModel.index({valueType: 1, space: 1, clusters: 1}, {unique: true}); // Define Cache schema. const Cache = new Schema({ @@ -259,7 +260,7 @@ module.exports.factory = function(mongoose) { writeBehindFlushThreadCount: Number, writeBehindCoalescing: {type: Boolean, default: true}, - invalidate: Boolean, + isInvalidate: Boolean, defaultLockTimeout: Number, atomicWriteOrderMode: {type: String, enum: ['CLOCK', 'PRIMARY']}, writeSynchronizationMode: {type: String, enum: ['FULL_SYNC', 'FULL_ASYNC', 'PRIMARY_SYNC']}, @@ -328,7 +329,7 @@ module.exports.factory = function(mongoose) { topologyValidator: String }); - Cache.index({name: 1, space: 1}, {unique: true}); + Cache.index({name: 1, space: 1, clusters: 1}, {unique: true}); const Igfs = new Schema({ space: {type: ObjectId, ref: 'Space', index: true, required: true}, @@ -376,7 +377,7 @@ module.exports.factory = function(mongoose) { updateFileLengthOnFlush: Boolean }); - Igfs.index({name: 1, space: 1}, {unique: true}); + Igfs.index({name: 1, space: 1, clusters: 1}, {unique: true}); // Define Cluster schema. @@ -586,6 +587,7 @@ module.exports.factory = function(mongoose) { compactFooter: Boolean }, caches: [{type: ObjectId, ref: 'Cache'}], + models: [{type: ObjectId, ref: 'DomainModel'}], clockSyncSamples: Number, clockSyncFrequency: Number, deploymentMode: {type: String, enum: ['PRIVATE', 'ISOLATED', 'SHARED', 'CONTINUOUS']}, http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/errors/AppErrorException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/AppErrorException.js b/modules/web-console/backend/errors/AppErrorException.js index 208b09b..19a9b0d 100644 --- a/modules/web-console/backend/errors/AppErrorException.js +++ b/modules/web-console/backend/errors/AppErrorException.js @@ -23,8 +23,6 @@ class AppErrorException extends Error { this.name = this.constructor.name; this.code = 400; - this.httpCode = 400; - this.message = message; if (typeof Error.captureStackTrace === 'function') Error.captureStackTrace(this, this.constructor); http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/errors/AuthFailedException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/AuthFailedException.js b/modules/web-console/backend/errors/AuthFailedException.js index 2772fad..9cab6ac 100644 --- a/modules/web-console/backend/errors/AuthFailedException.js +++ b/modules/web-console/backend/errors/AuthFailedException.js @@ -23,7 +23,7 @@ class AuthFailedException extends AppErrorException { constructor(message) { super(message); - this.httpCode = 401; + this.code = 401; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/errors/IllegalAccessError.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/IllegalAccessError.js b/modules/web-console/backend/errors/IllegalAccessError.js index bc07ef8..7de9bb1 100644 --- a/modules/web-console/backend/errors/IllegalAccessError.js +++ b/modules/web-console/backend/errors/IllegalAccessError.js @@ -22,7 +22,8 @@ const AppErrorException = require('./AppErrorException'); class IllegalAccessError extends AppErrorException { constructor(message) { super(message); - this.httpCode = 403; + + this.code = 403; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/errors/IllegalArgumentException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/IllegalArgumentException.js b/modules/web-console/backend/errors/IllegalArgumentException.js index 41ccd9b..aeb4187 100644 --- a/modules/web-console/backend/errors/IllegalArgumentException.js +++ b/modules/web-console/backend/errors/IllegalArgumentException.js @@ -22,7 +22,6 @@ const AppErrorException = require('./AppErrorException'); class IllegalArgumentException extends AppErrorException { constructor(message) { super(message); - this.httpCode = 400; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/errors/MissingResourceException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/MissingResourceException.js b/modules/web-console/backend/errors/MissingResourceException.js index bcfb408..aeac70e 100644 --- a/modules/web-console/backend/errors/MissingResourceException.js +++ b/modules/web-console/backend/errors/MissingResourceException.js @@ -23,7 +23,7 @@ class MissingResourceException extends AppErrorException { constructor(message) { super(message); - this.httpCode = 404; + this.code = 404; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/errors/ServerErrorException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/ServerErrorException.js b/modules/web-console/backend/errors/ServerErrorException.js index 439755e..c2edb7f 100644 --- a/modules/web-console/backend/errors/ServerErrorException.js +++ b/modules/web-console/backend/errors/ServerErrorException.js @@ -23,7 +23,6 @@ class ServerErrorException extends Error { this.name = this.constructor.name; this.code = 500; - this.httpCode = 500; this.message = message; if (typeof Error.captureStackTrace === 'function') http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/middlewares/api.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/middlewares/api.js b/modules/web-console/backend/middlewares/api.js index 23fd7ae..e901ec4 100644 --- a/modules/web-console/backend/middlewares/api.js +++ b/modules/web-console/backend/middlewares/api.js @@ -20,10 +20,11 @@ // Fire me up! module.exports = { - implements: 'middlewares:api' + implements: 'middlewares:api', + inject: ['require(lodash)'] }; -module.exports.factory = () => { +module.exports.factory = (_) => { return (req, res, next) => { // Set headers to avoid API caching in browser (esp. IE) res.header('Cache-Control', 'must-revalidate'); @@ -32,18 +33,16 @@ module.exports.factory = () => { res.api = { error(err) { - if (err.name === 'MongoError') + if (_.includes(['MongoError', 'MongooseError'], err.name)) return res.status(500).send(err.message); res.status(err.httpCode || err.code || 500).send(err.message); }, ok(data) { - res.status(200).json(data); - }, - serverError(err) { - err.httpCode = 500; + if (_.isNil(data)) + return res.sendStatus(404); - res.api.error(err); + res.status(200).json(data); } }; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/middlewares/demo.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/middlewares/demo.js b/modules/web-console/backend/middlewares/demo.js new file mode 100644 index 0000000..537ede1 --- /dev/null +++ b/modules/web-console/backend/middlewares/demo.js @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +// Fire me up! + +module.exports = { + implements: 'middlewares:demo', + factory: () => { + return (req, res, next) => { + req.demo = () => req.header('IgniteDemoMode'); + + next(); + }; + } +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/migrations/1502249492000-invalidate_rename.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1502249492000-invalidate_rename.js b/modules/web-console/backend/migrations/1502249492000-invalidate_rename.js new file mode 100644 index 0000000..479e694 --- /dev/null +++ b/modules/web-console/backend/migrations/1502249492000-invalidate_rename.js @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +exports.up = function up(done) { + this('Cache').update({}, { $rename: {invalidate: 'isInvalidate'}}, {multi: true}) + .then(() => done()) + .catch(done); +}; + +exports.down = function down(done) { + this('Cache').update({}, { $rename: {isInvalidate: 'invalidate'}}, {multi: true}) + .then(() => done()) + .catch(done); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/migrations/1502432624000-cache-index.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1502432624000-cache-index.js b/modules/web-console/backend/migrations/1502432624000-cache-index.js new file mode 100644 index 0000000..147e2ad --- /dev/null +++ b/modules/web-console/backend/migrations/1502432624000-cache-index.js @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const recreateIndex = require('./migration-utils').recreateIndex; + +exports.up = function up(done) { + recreateIndex(done, this('Cache').collection, + 'name_1_space_1', + {name: 1, space: 1}, + {name: 1, space: 1, clusters: 1}); +}; + +exports.down = function down(done) { + recreateIndex(done, this('Cache').collection, + 'name_1_space_1_clusters_1', + {name: 1, space: 1, clusters: 1}, + {name: 1, space: 1}); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/migrations/1504672035000-igfs-index.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1504672035000-igfs-index.js b/modules/web-console/backend/migrations/1504672035000-igfs-index.js new file mode 100644 index 0000000..e802ca9 --- /dev/null +++ b/modules/web-console/backend/migrations/1504672035000-igfs-index.js @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const recreateIndex = require('./migration-utils').recreateIndex; + +exports.up = function up(done) { + recreateIndex(done, this('Igfs').collection, + 'name_1_space_1', + {name: 1, space: 1}, + {name: 1, space: 1, clusters: 1}); +}; + +exports.down = function down(done) { + recreateIndex(done, this('Igfs').collection, + 'name_1_space_1_clusters_1', + {name: 1, space: 1, clusters: 1}, + {name: 1, space: 1}); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/migrations/1505114649000-models-index.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1505114649000-models-index.js b/modules/web-console/backend/migrations/1505114649000-models-index.js new file mode 100644 index 0000000..c007b01 --- /dev/null +++ b/modules/web-console/backend/migrations/1505114649000-models-index.js @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const recreateIndex = require('./migration-utils').recreateIndex; + +exports.up = function up(done) { + recreateIndex(done, this('DomainModel').collection, + 'valueType_1_space_1', + {valueType: 1, space: 1}, + {valueType: 1, space: 1, clusters: 1}); +}; + +exports.down = function down(done) { + recreateIndex(done, this('DomainModel').collection, + 'valueType_1_space_1_clusters_1', + {valueType: 1, space: 1, clusters: 1}, + {valueType: 1, space: 1}); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/migrations/1508395969410-init-registered-date.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1508395969410-init-registered-date.js b/modules/web-console/backend/migrations/1508395969410-init-registered-date.js index b994cac..2f1018f 100644 --- a/modules/web-console/backend/migrations/1508395969410-init-registered-date.js +++ b/modules/web-console/backend/migrations/1508395969410-init-registered-date.js @@ -18,10 +18,11 @@ const _ = require('lodash'); exports.up = function up(done) { - const accounts = this('Account'); + const accountsModel = this('Account'); - accounts.find({}).lean().exec() - .then((data) => _.forEach(data, (acc) => accounts.update({_id: acc._id}, {$set: {registered: acc.lastLogin}}, {upsert: true}).exec())) + accountsModel.find({}).lean().exec() + .then((accounts) => _.reduce(accounts, (start, account) => start + .then(() => accountsModel.update({_id: account._id}, {$set: {registered: account.lastLogin}}).exec()), Promise.resolve())) .then(() => done()) .catch(done); }; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/migrations/1516948939797-migrate-configs.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1516948939797-migrate-configs.js b/modules/web-console/backend/migrations/1516948939797-migrate-configs.js new file mode 100644 index 0000000..cfaf0f2 --- /dev/null +++ b/modules/web-console/backend/migrations/1516948939797-migrate-configs.js @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const _ = require('lodash'); + +const log = require('./migration-utils').log; +const error = require('./migration-utils').error; + +const getClusterForMigration = require('./migration-utils').getClusterForMigration; +const getCacheForMigration = require('./migration-utils').getCacheForMigration; + +const _debug = false; +const DUPLICATE_KEY_ERROR = 11000; + +function linkCacheToCluster(clustersModel, cluster, cachesModel, cache, domainsModel) { + return clustersModel.update({_id: cluster._id}, {$addToSet: {caches: cache._id}}).exec() + .then(() => cachesModel.update({_id: cache._id}, {clusters: [cluster._id]}).exec()) + .then(() => { + if (_.isEmpty(cache.domains)) + return Promise.resolve(); + + return _.reduce(cache.domains, (start, domain) => start.then(() => { + return domainsModel.update({_id: domain}, {clusters: [cluster._id]}).exec() + .then(() => clustersModel.update({_id: cluster._id}, {$addToSet: {models: domain}}).exec()); + }), Promise.resolve()); + }) + .catch((err) => error(`Failed link cache to cluster [cache=${cache.name}, cluster=${cluster.name}]`, err)); +} + +function cloneCache(clustersModel, cachesModel, domainsModel, cache) { + const cacheId = cache._id; + const clusters = cache.clusters; + + cache.clusters = []; + + if (cache.cacheStoreFactory && cache.cacheStoreFactory.kind === null) + delete cache.cacheStoreFactory.kind; + + return _.reduce(clusters, (start, cluster, idx) => start.then(() => { + if (idx > 0) { + delete cache._id; + + const newCache = _.clone(cache); + + newCache.clusters = [cluster]; + + return clustersModel.update({_id: {$in: newCache.clusters}}, {$pull: {caches: cacheId}}, {multi: true}).exec() + .then(() => cachesModel.create(newCache)) + .catch((err) => { + if (err.code === DUPLICATE_KEY_ERROR) { + log(`Failed to clone cache, will change cache name and retry [cache=${newCache.name}]`); + newCache.name += '_dup'; + + return cachesModel.create(newCache); + } + + return Promise.reject(err); + }) + .then((clone) => clustersModel.update({_id: {$in: newCache.clusters}}, {$addToSet: {caches: clone._id}}, {multi: true}).exec() + .then(() => clone)) + .then((clone) => { + const domainIds = newCache.domains; + + if (_.isEmpty(domainIds)) + return Promise.resolve(); + + return _.reduce(domainIds, (start, domainId) => start.then(() => { + return domainsModel.findOne({_id: domainId}).lean().exec() + .then((domain) => { + delete domain._id; + + const newDomain = _.clone(domain); + + newDomain.caches = [clone._id]; + newDomain.clusters = [cluster]; + + return domainsModel.create(newDomain) + .catch((err) => { + if (err.code === DUPLICATE_KEY_ERROR) { + log(`Failed to clone domain, will change type name and retry [cache=${newCache.name}, valueType=${newDomain.valueType}]`); + newDomain.valueType += '_dup'; + + return domainsModel.create(newDomain); + } + }) + .then((createdDomain) => clustersModel.update({_id: cluster}, {$addToSet: {models: createdDomain._id}}).exec()) + .catch((err) => error('Failed to clone domain', err)); + }) + .catch((err) => error(`Failed to duplicate domain model[domain=${domainId}], cache=${clone.name}]`, err)); + }), Promise.resolve()); + }) + .catch((err) => error(`Failed to clone cache[id=${cacheId}, name=${cache.name}]`, err)); + } + + return cachesModel.update({_id: cacheId}, {clusters: [cluster]}).exec() + .then(() => clustersModel.update({_id: cluster}, {$addToSet: {models: {$each: cache.domains}}}).exec()); + }), Promise.resolve()); +} + +function migrateCache(clustersModel, cachesModel, domainsModel, cache) { + const len = _.size(cache.clusters); + + if (len < 1) { + if (_debug) + log(`Found cache not linked to cluster [cache=${cache.name}]`); + + return getClusterForMigration(clustersModel, cache.space) + .then((clusterLostFound) => linkCacheToCluster(clustersModel, clusterLostFound, cachesModel, cache, domainsModel)); + } + + if (len > 1) { + if (_debug) + log(`Found cache linked to many clusters [cache=${cache.name}, cnt=${len}]`); + + return cloneCache(clustersModel, cachesModel, domainsModel, cache); + } + + // Nothing to migrate, cache linked to cluster 1-to-1. + return Promise.resolve(); +} + +function migrateCaches(clustersModel, cachesModel, domainsModel) { + return cachesModel.find({}).lean().exec() + .then((caches) => { + const sz = _.size(caches); + + if (sz > 0) { + log(`Caches to migrate: ${sz}`); + + return _.reduce(caches, (start, cache) => start.then(() => migrateCache(clustersModel, cachesModel, domainsModel, cache)), Promise.resolve()) + .then(() => log('Caches migration finished.')); + } + + return Promise.resolve(); + + }) + .catch((err) => error('Caches migration failed', err)); +} + +function linkIgfsToCluster(clustersModel, cluster, igfsModel, igfs) { + return clustersModel.update({_id: cluster._id}, {$addToSet: {igfss: igfs._id}}).exec() + .then(() => igfsModel.update({_id: igfs._id}, {clusters: [cluster._id]}).exec()) + .catch((err) => error(`Failed link IGFS to cluster [IGFS=${igfs.name}, cluster=${cluster.name}]`, err)); +} + +function cloneIgfs(clustersModel, igfsModel, igfs) { + const igfsId = igfs._id; + const clusters = igfs.clusters; + + delete igfs._id; + igfs.clusters = []; + + return _.reduce(clusters, (start, cluster, idx) => start.then(() => { + const newIgfs = _.clone(igfs); + + newIgfs.clusters = [cluster]; + + if (idx > 0) { + return clustersModel.update({_id: {$in: newIgfs.clusters}}, {$pull: {igfss: igfsId}}, {multi: true}).exec() + .then(() => igfsModel.create(newIgfs)) + .then((clone) => clustersModel.update({_id: {$in: newIgfs.clusters}}, {$addToSet: {igfss: clone._id}}, {multi: true}).exec()) + .catch((err) => error(`Failed to clone IGFS: id=${igfsId}, name=${igfs.name}]`, err)); + } + + return igfsModel.update({_id: igfsId}, {clusters: [cluster]}).exec(); + }), Promise.resolve()); +} + +function migrateIgfs(clustersModel, igfsModel, igfs) { + const len = _.size(igfs.clusters); + + if (len < 1) { + if (_debug) + log(`Found IGFS not linked to cluster [IGFS=${igfs.name}]`); + + return getClusterForMigration(clustersModel, igfs.space) + .then((clusterLostFound) => linkIgfsToCluster(clustersModel, clusterLostFound, igfsModel, igfs)); + } + + if (len > 1) { + if (_debug) + log(`Found IGFS linked to many clusters [IGFS=${igfs.name}, cnt=${len}]`); + + return cloneIgfs(clustersModel, igfsModel, igfs); + } + + // Nothing to migrate, IGFS linked to cluster 1-to-1. + return Promise.resolve(); +} + +function migrateIgfss(clustersModel, igfsModel) { + return igfsModel.find({}).lean().exec() + .then((igfss) => { + const sz = _.size(igfss); + + if (sz > 0) { + log(`IGFS to migrate: ${sz}`); + + return _.reduce(igfss, (start, igfs) => start.then(() => migrateIgfs(clustersModel, igfsModel, igfs)), Promise.resolve()) + .then(() => log('IGFS migration finished.')); + } + + return Promise.resolve(); + }) + .catch((err) => error('IGFS migration failed', err)); +} + +function linkDomainToCluster(clustersModel, cluster, domainsModel, domain) { + return clustersModel.update({_id: cluster._id}, {$addToSet: {models: domain._id}}).exec() + .then(() => domainsModel.update({_id: domain._id}, {clusters: [cluster._id]}).exec()) + .catch((err) => error(`Failed link domain model to cluster [domain=${domain._id}, cluster=${cluster.name}]`, err)); +} + +function linkDomainToCache(cachesModel, cache, domainsModel, domain) { + return cachesModel.update({_id: cache._id}, {$addToSet: {domains: domain._id}}).exec() + .then(() => domainsModel.update({_id: domain._id}, {caches: [cache._id]}).exec()) + .catch((err) => error(`Failed link domain model to cache[cache=${cache.name}, domain=${domain._id}]`, err)); +} + +function migrateDomain(clustersModel, cachesModel, domainsModel, domain) { + if (_.isEmpty(domain.caches)) { + if (_debug) + log(`Found domain model not linked to cache [domain=${domain._id}]`); + + return getClusterForMigration(clustersModel, domain.space) + .then((clusterLostFound) => linkDomainToCluster(clustersModel, clusterLostFound, domainsModel, domain)) + .then(() => getCacheForMigration(clustersModel, cachesModel, domain.space)) + .then((cacheLostFound) => linkDomainToCache(cachesModel, cacheLostFound, domainsModel, domain)) + .catch((err) => error(`Failed to migrate not linked domain [domain=${domain._id}]`, err)); + } + + if (_.isEmpty(domain.clusters)) { + return cachesModel.findOne({_id: {$in: domain.caches}}).lean().exec() + .then((cache) => { + if (cache) { + const clusterId = cache.clusters[0]; + + return domainsModel.update({_id: domain._id}, {clusters: [clusterId]}).exec() + .then(() => clustersModel.update({_id: clusterId}, {$addToSet: {models: domain._id}}).exec()); + } + + log(`Found broken domain: [domain=${domain._id}, caches=${domain.caches}]`); + + return Promise.resolve(); + }) + .catch((err) => error(`Failed to migrate domain [domain=${domain._id}]`, err)); + } + + // Nothing to migrate, other domains will be migrated with caches. + return Promise.resolve(); +} + +function migrateDomains(clustersModel, cachesModel, domainsModel) { + return domainsModel.find({}).lean().exec() + .then((domains) => { + const sz = _.size(domains); + + if (sz > 0) { + log(`Domain models to migrate: ${sz}`); + + return _.reduce(domains, (start, domain) => start.then(() => migrateDomain(clustersModel, cachesModel, domainsModel, domain)), Promise.resolve()) + .then(() => log('Domain models migration finished.')); + } + + return Promise.resolve(); + }) + .catch((err) => error('Domain models migration failed', err)); +} + +function deduplicate(title, model, name) { + return model.find({}).lean().exec() + .then((items) => { + const sz = _.size(items); + + if (sz > 0) { + log(`Deduplication of ${title} started...`); + + let cnt = 0; + + return _.reduce(items, (start, item) => start.then(() => { + const data = item[name]; + + const dataSz = _.size(data); + + if (dataSz < 2) + return Promise.resolve(); + + const deduped = _.uniqWith(data, _.isEqual); + + if (dataSz !== _.size(deduped)) { + return model.updateOne({_id: item._id}, {$set: {[name]: deduped}}) + .then(() => cnt++); + } + + return Promise.resolve(); + }), Promise.resolve()) + .then(() => log(`Deduplication of ${title} finished: ${cnt}.`)); + } + + return Promise.resolve(); + }); +} + +exports.up = function up(done) { + const clustersModel = this('Cluster'); + const cachesModel = this('Cache'); + const domainsModel = this('DomainModel'); + const igfsModel = this('Igfs'); + + process.on('unhandledRejection', function(reason, p) { + console.log('Unhandled rejection at:', p, 'reason:', reason); + }); + + Promise.resolve() + .then(() => deduplicate('Cluster caches', clustersModel, 'caches')) + .then(() => deduplicate('Cluster IGFS', clustersModel, 'igfss')) + .then(() => deduplicate('Cache clusters', cachesModel, 'clusters')) + .then(() => deduplicate('Cache domains', cachesModel, 'domains')) + .then(() => deduplicate('IGFS clusters', igfsModel, 'clusters')) + .then(() => deduplicate('Domain model caches', domainsModel, 'caches')) + .then(() => migrateCaches(clustersModel, cachesModel, domainsModel)) + .then(() => migrateIgfss(clustersModel, igfsModel)) + .then(() => migrateDomains(clustersModel, cachesModel, domainsModel)) + .then(() => done()) + .catch(done); +}; + +exports.down = function down(done) { + log('Model migration can not be reverted'); + + done(); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/migrations/migration-utils.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/migration-utils.js b/modules/web-console/backend/migrations/migration-utils.js new file mode 100644 index 0000000..0397247 --- /dev/null +++ b/modules/web-console/backend/migrations/migration-utils.js @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function log(msg) { + console.log(`[${new Date().toISOString()}] [INFO ] ${msg}`); +} + +function error(msg, err) { + console.log(`[${new Date().toISOString()}] [ERROR] ${msg}. Error: ${err}`); +} + +function recreateIndex0(done, model, oldIdxName, oldIdx, newIdx) { + return model.indexExists(oldIdxName) + .then((exists) => { + if (exists) { + return model.dropIndex(oldIdx) + .then(() => model.createIndex(newIdx, {unique: true, background: false})); + } + }) + .then(() => done()) + .catch((err) => { + if (err.code === 12587) { + log(`Background operation in progress for: ${oldIdxName}, will retry in 3 seconds.`); + + setTimeout(() => recreateIndex0(done, model, oldIdxName, oldIdx, newIdx), 3000); + } + else { + log(`Failed to recreate index: ${err}`); + + done(); + } + }); +} + +function recreateIndex(done, model, oldIdxName, oldIdx, newIdx) { + setTimeout(() => recreateIndex0(done, model, oldIdxName, oldIdx, newIdx), 1000); +} + +const LOST_AND_FOUND = 'LOST_AND_FOUND'; + +let _clusterLostAndFound = null; + +function getClusterForMigration(clustersModel, space) { + if (_clusterLostAndFound) + return Promise.resolve(_clusterLostAndFound); + + return clustersModel.findOne({name: LOST_AND_FOUND}).lean().exec() + .then((cluster) => { + if (cluster) { + _clusterLostAndFound = cluster; + + return cluster; + } + + return clustersModel.create({ + space, + name: LOST_AND_FOUND, + connector: {noDelay: true}, + communication: {tcpNoDelay: true}, + igfss: [], + caches: [], + binaryConfiguration: { + compactFooter: true, + typeConfigurations: [] + }, + discovery: { + kind: 'Multicast', + Multicast: {addresses: ['127.0.0.1:47500..47510']}, + Vm: {addresses: ['127.0.0.1:47500..47510']} + } + }) + .then((cluster) => { + _clusterLostAndFound = cluster; + + return cluster; + }); + }); +} + +let _cacheLostAndFound = null; + +function getCacheForMigration(clustersModel, cachesModel, space) { + if (_cacheLostAndFound) + return Promise.resolve(_cacheLostAndFound); + + return cachesModel.findOne({name: LOST_AND_FOUND}) + .then((cache) => { + if (cache) { + _cacheLostAndFound = cache; + + return cache; + } + + return getClusterForMigration(clustersModel, space) + .then((cluster) => { + return cachesModel.create({ + space, + name: LOST_AND_FOUND, + clusters: [cluster._id], + domains: [], + cacheMode: 'PARTITIONED', + atomicityMode: 'ATOMIC', + readFromBackup: true, + copyOnRead: true, + readThrough: false, + writeThrough: false, + sqlFunctionClasses: [], + writeBehindCoalescing: true, + cacheStoreFactory: { + CacheHibernateBlobStoreFactory: {hibernateProperties: []}, + CacheJdbcBlobStoreFactory: {connectVia: 'DataSource'} + }, + nearConfiguration: {}, + evictionPolicy: {} + }); + }) + .then((cache) => { + return clustersModel.update({_id: cache.clusters[0]}, {$addToSet: {caches: cache._id}}).exec() + .then(() => cache); + }) + .then((cache) => { + _cacheLostAndFound = cache; + + return cache; + }); + }); +} + +module.exports = { + log, + error, + recreateIndex, + getClusterForMigration, + getCacheForMigration +}; + + + + http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/migrations/recreate-index.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/recreate-index.js b/modules/web-console/backend/migrations/recreate-index.js deleted file mode 100644 index 328ed43..0000000 --- a/modules/web-console/backend/migrations/recreate-index.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -module.exports = function(done, model, oldIdxName, oldIdx, newIdx) { - model.indexExists(oldIdxName) - .then((exists) => { - if (exists) { - return model.dropIndex(oldIdx) - .then(() => model.createIndex(newIdx, {unique: true})); - } - }) - .then(() => done()) - .catch(done); -}; http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/routes/caches.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/caches.js b/modules/web-console/backend/routes/caches.js index d7ed8b8..25f76a1 100644 --- a/modules/web-console/backend/routes/caches.js +++ b/modules/web-console/backend/routes/caches.js @@ -30,6 +30,18 @@ module.exports.factory = function(mongo, cachesService) { return new Promise((factoryResolve) => { const router = new express.Router(); + router.get('/:_id', (req, res) => { + cachesService.get(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.delete('/', (req, res) => { + cachesService.remove(req.body.ids) + .then(res.api.ok) + .catch(res.api.error); + }); + /** * Save cache. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/routes/clusters.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/clusters.js b/modules/web-console/backend/routes/clusters.js index 24334c2..ac7b25e 100644 --- a/modules/web-console/backend/routes/clusters.js +++ b/modules/web-console/backend/routes/clusters.js @@ -23,13 +23,55 @@ const express = require('express'); module.exports = { implements: 'routes/clusters', - inject: ['mongo', 'services/clusters'] + inject: ['mongo', 'services/clusters', 'services/caches', 'services/domains', 'services/igfss'] }; -module.exports.factory = function(mongo, clustersService) { +module.exports.factory = function(mongo, clustersService, cachesService, domainsService, igfssService) { return new Promise((factoryResolve) => { const router = new express.Router(); + router.get('/:_id/caches', (req, res) => { + cachesService.shortList(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.get('/:_id/models', (req, res) => { + domainsService.shortList(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.get('/:_id/igfss', (req, res) => { + igfssService.shortList(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.get('/:_id', (req, res) => { + clustersService.get(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.get('/', (req, res) => { + clustersService.shortList(req.currentUserId(), req.demo()) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.put('/basic', (req, res) => { + clustersService.upsertBasic(req.currentUserId(), req.demo(), req.body) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.put('/', (req, res) => { + clustersService.upsert(req.currentUserId(), req.demo(), req.body) + .then(res.api.ok) + .catch(res.api.error); + }); + /** * Save cluster. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/routes/configuration.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/configuration.js b/modules/web-console/backend/routes/configuration.js index d9bde75..9ec002b 100644 --- a/modules/web-console/backend/routes/configuration.js +++ b/modules/web-console/backend/routes/configuration.js @@ -29,11 +29,21 @@ module.exports = { module.exports.factory = function(mongo, configurationsService) { return new Promise((factoryResolve) => { const router = new express.Router(); + /** * Get all user configuration in current space. */ router.get('/list', (req, res) => { - configurationsService.list(req.currentUserId(), req.header('IgniteDemoMode')) + configurationsService.list(req.currentUserId(), req.demo()) + .then(res.api.ok) + .catch(res.api.error); + }); + + /** + * Get user configuration in current space. + */ + router.get('/:_id', (req, res) => { + configurationsService.get(req.currentUserId(), req.demo(), req.params._id) .then(res.api.ok) .catch(res.api.error); }); http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/routes/domains.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/domains.js b/modules/web-console/backend/routes/domains.js index caa9201..9360421 100644 --- a/modules/web-console/backend/routes/domains.js +++ b/modules/web-console/backend/routes/domains.js @@ -30,6 +30,12 @@ module.exports.factory = (mongo, domainsService) => { return new Promise((factoryResolve) => { const router = new express.Router(); + router.get('/:_id', (req, res) => { + domainsService.get(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + /** * Save domain model. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/routes/igfss.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/igfss.js b/modules/web-console/backend/routes/igfss.js index b95f21f..c975249 100644 --- a/modules/web-console/backend/routes/igfss.js +++ b/modules/web-console/backend/routes/igfss.js @@ -30,6 +30,18 @@ module.exports.factory = function(mongo, igfssService) { return new Promise((factoryResolve) => { const router = new express.Router(); + router.get('/:_id', (req, res) => { + igfssService.get(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.delete('/', (req, res) => { + igfssService.remove(req.body.ids) + .then(res.api.ok) + .catch(res.api.error); + }); + /** * Save IGFS. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/7ee1683e/modules/web-console/backend/services/caches.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/services/caches.js b/modules/web-console/backend/services/caches.js index 9cb65a1..3d0baee 100644 --- a/modules/web-console/backend/services/caches.js +++ b/modules/web-console/backend/services/caches.js @@ -28,11 +28,11 @@ module.exports = { /** * @param mongo - * @param {SpacesService} spaceService + * @param {SpacesService} spacesService * @param errors * @returns {CachesService} */ -module.exports.factory = (mongo, spaceService, errors) => { +module.exports.factory = (mongo, spacesService, errors) => { /** * Convert remove status operation to own presentation. * @@ -101,6 +101,55 @@ module.exports.factory = (mongo, spaceService, errors) => { * Service for manipulate Cache entities. */ class CachesService { + static shortList(userId, demo, clusterId) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.Cache.find({space: {$in: spaceIds}, clusters: clusterId }).select('name cacheMode atomicityMode backups').lean().exec()); + } + + static get(userId, demo, _id) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.Cache.findOne({space: {$in: spaceIds}, _id}).lean().exec()); + } + + static upsertBasic(cache) { + if (_.isNil(cache._id)) + return Promise.reject(new errors.IllegalArgumentException('Cache id can not be undefined or null')); + + const query = _.pick(cache, ['space', '_id']); + const newDoc = _.pick(cache, ['space', '_id', 'name', 'cacheMode', 'atomicityMode', 'backups', 'clusters']); + + return mongo.Cache.update(query, {$set: newDoc}, {upsert: true}).exec() + .catch((err) => { + if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) + throw new errors.DuplicateKeyException(`Cache with name: "${cache.name}" already exist.`); + + throw err; + }) + .then((updated) => { + if (updated.nModified === 0) + return mongo.Cache.update(query, {$set: cache}, {upsert: true}).exec(); + + return updated; + }); + } + + static upsert(cache) { + if (_.isNil(cache._id)) + return Promise.reject(new errors.IllegalArgumentException('Cache id can not be undefined or null')); + + const query = _.pick(cache, ['space', '_id']); + + return mongo.Cache.update(query, {$set: cache}, {upsert: true}).exec() + .then(() => mongo.DomainModel.update({_id: {$in: cache.domains}}, {$addToSet: {caches: cache._id}}, {multi: true}).exec()) + .then(() => mongo.DomainModel.update({_id: {$nin: cache.domains}}, {$pull: {caches: cache._id}}, {multi: true}).exec()) + .catch((err) => { + if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) + throw new errors.DuplicateKeyException(`Cache with name: "${cache.name}" already exist.`); + + throw err; + }); + } + /** * Create or update cache. * @@ -125,21 +174,26 @@ module.exports.factory = (mongo, spaceService, errors) => { } /** - * Remove cache. + * Remove caches. * - * @param {mongo.ObjectId|String} cacheId - The cache id for remove. + * @param {Array.<String>|String} ids - The cache ids for remove. * @returns {Promise.<{rowsAffected}>} - The number of affected rows. */ - static remove(cacheId) { - if (_.isNil(cacheId)) + static remove(ids) { + if (_.isNil(ids)) return Promise.reject(new errors.IllegalArgumentException('Cache id can not be undefined or null')); - return mongo.Cluster.update({caches: {$in: [cacheId]}}, {$pull: {caches: cacheId}}, {multi: true}).exec() - .then(() => mongo.Cluster.update({}, {$pull: {checkpointSpi: {kind: 'Cache', Cache: {cache: cacheId}}}}, {multi: true}).exec()) - // TODO WC-201 fix clenup of cache on deletion for cluster service configuration. + ids = _.castArray(ids); + + if (_.isEmpty(ids)) + return Promise.resolve({rowsAffected: 0}); + + return mongo.Cluster.update({caches: {$in: ids}}, {$pull: {caches: {$in: ids}}}, {multi: true}).exec() + .then(() => mongo.Cluster.update({}, {$pull: {checkpointSpi: {kind: 'Cache', Cache: {cache: {$in: ids}}}}}, {multi: true}).exec()) + // TODO WC-201 fix cleanup of cache on deletion for cluster service configuration. // .then(() => mongo.Cluster.update({'serviceConfigurations.cache': cacheId}, {$unset: {'serviceConfigurations.$.cache': ''}}, {multi: true}).exec()) - .then(() => mongo.DomainModel.update({caches: {$in: [cacheId]}}, {$pull: {caches: cacheId}}, {multi: true}).exec()) - .then(() => mongo.Cache.remove({_id: cacheId}).exec()) + .then(() => mongo.DomainModel.update({caches: {$in: ids}}, {$pull: {caches: {$in: ids}}}, {multi: true}).exec()) + .then(() => mongo.Cache.remove({_id: {$in: ids}}).exec()) .then(convertRemoveStatus); } @@ -151,7 +205,7 @@ module.exports.factory = (mongo, spaceService, errors) => { * @returns {Promise.<{rowsAffected}>} - The number of affected rows. */ static removeAll(userId, demo) { - return spaceService.spaceIds(userId, demo) + return spacesService.spaceIds(userId, demo) .then(removeAllBySpaces) .then(convertRemoveStatus); }
