IGNITE-5737 Implemented migration to cluster centric model.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f173be81 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f173be81 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f173be81 Branch: refs/heads/ignite-5737-2 Commit: f173be81fc0fd7ebea74dfe3b3824582306f81da Parents: 4abcb31 Author: Alexey Kuznetsov <[email protected]> Authored: Fri Feb 9 11:02:46 2018 +0700 Committer: Alexey Kuznetsov <[email protected]> Committed: Fri Feb 9 11:02:46 2018 +0700 ---------------------------------------------------------------------- modules/web-console/backend/app/apiServer.js | 9 +- modules/web-console/backend/app/routes.js | 5 +- modules/web-console/backend/app/schemas.js | 8 +- .../backend/errors/AppErrorException.js | 2 - .../backend/errors/AuthFailedException.js | 2 +- .../backend/errors/IllegalAccessError.js | 3 +- .../backend/errors/IllegalArgumentException.js | 1 - .../backend/errors/MissingResourceException.js | 2 +- .../backend/errors/ServerErrorException.js | 1 - modules/web-console/backend/middlewares/api.js | 15 +- modules/web-console/backend/middlewares/demo.js | 31 ++ .../1502249492000-invalidate_rename.js | 28 ++ .../migrations/1502432624000-cache-index.js | 32 ++ .../migrations/1504672035000-igfs-index.js | 32 ++ .../migrations/1505114649000-models-index.js | 32 ++ .../1508395969410-init-registered-date.js | 8 +- .../migrations/1516948939797-migrate-configs.js | 289 +++++++++++++++++++ .../backend/migrations/migration-utils.js | 121 ++++++++ .../backend/migrations/recreate-index.js | 30 -- modules/web-console/backend/routes/caches.js | 12 + modules/web-console/backend/routes/clusters.js | 46 ++- .../web-console/backend/routes/configuration.js | 12 +- modules/web-console/backend/routes/domains.js | 6 + modules/web-console/backend/routes/igfss.js | 12 + modules/web-console/backend/services/caches.js | 78 ++++- .../web-console/backend/services/clusters.js | 153 ++++++++-- .../backend/services/configurations.js | 12 + modules/web-console/backend/services/domains.js | 96 +++++- modules/web-console/backend/services/igfss.js | 44 ++- .../web-console/backend/services/sessions.js | 2 +- modules/web-console/backend/services/spaces.js | 2 +- .../backend/test/unit/CacheService.test.js | 45 ++- .../backend/test/unit/ClusterService.test.js | 233 ++++++++++++++- .../backend/test/unit/DomainService.test.js | 5 + modules/web-console/frontend/app/app.js | 6 +- .../page-configure-basic/template.pug | 2 +- .../generator/ConfigurationGenerator.js | 2 +- .../generator/PlatformGenerator.js | 2 +- .../states/configuration/caches/general.pug | 2 +- .../states/configuration/domains/query.pug | 6 + 40 files changed, 1295 insertions(+), 134 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/app/apiServer.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/app/apiServer.js b/modules/web-console/backend/app/apiServer.js index 0030529..90c39ba 100644 --- a/modules/web-console/backend/app/apiServer.js +++ b/modules/web-console/backend/app/apiServer.js @@ -60,12 +60,11 @@ module.exports = { } // Catch 404 and forward to error handler. - app.use((req, res, next) => { - const err = new Error('Not Found: ' + req.originalUrl); + app.use((req, res) => { + if (req.xhr) + return res.status(404).send({ error: 'Not Found: ' + req.originalUrl }); - err.status = 404; - - next(err); + return res.sendStatus(404); }); // Production error handler: no stacktraces leaked to user. http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/app/routes.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/app/routes.js b/modules/web-console/backend/app/routes.js index aa6efae..ce7b5d8 100644 --- a/modules/web-console/backend/app/routes.js +++ b/modules/web-console/backend/app/routes.js @@ -43,7 +43,8 @@ module.exports.factory = function(publicRoute, adminRoute, profilesRoute, demoRo res.status(401).send('Access denied. You are not authorized to access this page.'); }; - // Registering the standard routes + // Registering the standard routes. + // NOTE: Order is important! app.use('/api/v1/', publicRoute); app.use('/api/v1/admin', _mustAuthenticated, _adminOnly, adminRoute); app.use('/api/v1/profile', _mustAuthenticated, profilesRoute); @@ -51,11 +52,11 @@ module.exports.factory = function(publicRoute, adminRoute, profilesRoute, demoRo app.all('/api/v1/configuration/*', _mustAuthenticated); - app.use('/api/v1/configuration', configurationsRoute); app.use('/api/v1/configuration/clusters', clustersRoute); app.use('/api/v1/configuration/domains', domainsRoute); app.use('/api/v1/configuration/caches', cachesRoute); app.use('/api/v1/configuration/igfs', igfssRoute); + app.use('/api/v1/configuration', configurationsRoute); app.use('/api/v1/notebooks', _mustAuthenticated, notebooksRoute); app.use('/api/v1/downloads', _mustAuthenticated, downloadsRoute); http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/app/schemas.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/app/schemas.js b/modules/web-console/backend/app/schemas.js index 0ed85b2..7a00c99 100644 --- a/modules/web-console/backend/app/schemas.js +++ b/modules/web-console/backend/app/schemas.js @@ -92,6 +92,7 @@ module.exports.factory = function(mongoose) { // Define Domain model schema. const DomainModel = new Schema({ space: {type: ObjectId, ref: 'Space', index: true, required: true}, + clusters: [{type: ObjectId, ref: 'Cluster'}], caches: [{type: ObjectId, ref: 'Cache'}], queryMetadata: {type: String, enum: ['Annotations', 'Configuration']}, kind: {type: String, enum: ['query', 'store', 'both']}, @@ -259,7 +260,7 @@ module.exports.factory = function(mongoose) { writeBehindFlushThreadCount: Number, writeBehindCoalescing: {type: Boolean, default: true}, - invalidate: Boolean, + isInvalidate: Boolean, defaultLockTimeout: Number, atomicWriteOrderMode: {type: String, enum: ['CLOCK', 'PRIMARY']}, writeSynchronizationMode: {type: String, enum: ['FULL_SYNC', 'FULL_ASYNC', 'PRIMARY_SYNC']}, @@ -328,7 +329,7 @@ module.exports.factory = function(mongoose) { topologyValidator: String }); - Cache.index({name: 1, space: 1}, {unique: true}); + Cache.index({name: 1, space: 1, clusters: 1}, {unique: true}); const Igfs = new Schema({ space: {type: ObjectId, ref: 'Space', index: true, required: true}, @@ -376,7 +377,7 @@ module.exports.factory = function(mongoose) { updateFileLengthOnFlush: Boolean }); - Igfs.index({name: 1, space: 1}, {unique: true}); + Igfs.index({name: 1, space: 1, clusters: 1}, {unique: true}); // Define Cluster schema. @@ -576,6 +577,7 @@ module.exports.factory = function(mongoose) { compactFooter: Boolean }, caches: [{type: ObjectId, ref: 'Cache'}], + models: [{type: ObjectId, ref: 'DomainModel'}], clockSyncSamples: Number, clockSyncFrequency: Number, deploymentMode: {type: String, enum: ['PRIVATE', 'ISOLATED', 'SHARED', 'CONTINUOUS']}, http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/errors/AppErrorException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/AppErrorException.js b/modules/web-console/backend/errors/AppErrorException.js index 208b09b..19a9b0d 100644 --- a/modules/web-console/backend/errors/AppErrorException.js +++ b/modules/web-console/backend/errors/AppErrorException.js @@ -23,8 +23,6 @@ class AppErrorException extends Error { this.name = this.constructor.name; this.code = 400; - this.httpCode = 400; - this.message = message; if (typeof Error.captureStackTrace === 'function') Error.captureStackTrace(this, this.constructor); http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/errors/AuthFailedException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/AuthFailedException.js b/modules/web-console/backend/errors/AuthFailedException.js index 2772fad..9cab6ac 100644 --- a/modules/web-console/backend/errors/AuthFailedException.js +++ b/modules/web-console/backend/errors/AuthFailedException.js @@ -23,7 +23,7 @@ class AuthFailedException extends AppErrorException { constructor(message) { super(message); - this.httpCode = 401; + this.code = 401; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/errors/IllegalAccessError.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/IllegalAccessError.js b/modules/web-console/backend/errors/IllegalAccessError.js index bc07ef8..7de9bb1 100644 --- a/modules/web-console/backend/errors/IllegalAccessError.js +++ b/modules/web-console/backend/errors/IllegalAccessError.js @@ -22,7 +22,8 @@ const AppErrorException = require('./AppErrorException'); class IllegalAccessError extends AppErrorException { constructor(message) { super(message); - this.httpCode = 403; + + this.code = 403; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/errors/IllegalArgumentException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/IllegalArgumentException.js b/modules/web-console/backend/errors/IllegalArgumentException.js index 41ccd9b..aeb4187 100644 --- a/modules/web-console/backend/errors/IllegalArgumentException.js +++ b/modules/web-console/backend/errors/IllegalArgumentException.js @@ -22,7 +22,6 @@ const AppErrorException = require('./AppErrorException'); class IllegalArgumentException extends AppErrorException { constructor(message) { super(message); - this.httpCode = 400; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/errors/MissingResourceException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/MissingResourceException.js b/modules/web-console/backend/errors/MissingResourceException.js index bcfb408..aeac70e 100644 --- a/modules/web-console/backend/errors/MissingResourceException.js +++ b/modules/web-console/backend/errors/MissingResourceException.js @@ -23,7 +23,7 @@ class MissingResourceException extends AppErrorException { constructor(message) { super(message); - this.httpCode = 404; + this.code = 404; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/errors/ServerErrorException.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/errors/ServerErrorException.js b/modules/web-console/backend/errors/ServerErrorException.js index 439755e..c2edb7f 100644 --- a/modules/web-console/backend/errors/ServerErrorException.js +++ b/modules/web-console/backend/errors/ServerErrorException.js @@ -23,7 +23,6 @@ class ServerErrorException extends Error { this.name = this.constructor.name; this.code = 500; - this.httpCode = 500; this.message = message; if (typeof Error.captureStackTrace === 'function') http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/middlewares/api.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/middlewares/api.js b/modules/web-console/backend/middlewares/api.js index 23fd7ae..e901ec4 100644 --- a/modules/web-console/backend/middlewares/api.js +++ b/modules/web-console/backend/middlewares/api.js @@ -20,10 +20,11 @@ // Fire me up! module.exports = { - implements: 'middlewares:api' + implements: 'middlewares:api', + inject: ['require(lodash)'] }; -module.exports.factory = () => { +module.exports.factory = (_) => { return (req, res, next) => { // Set headers to avoid API caching in browser (esp. IE) res.header('Cache-Control', 'must-revalidate'); @@ -32,18 +33,16 @@ module.exports.factory = () => { res.api = { error(err) { - if (err.name === 'MongoError') + if (_.includes(['MongoError', 'MongooseError'], err.name)) return res.status(500).send(err.message); res.status(err.httpCode || err.code || 500).send(err.message); }, ok(data) { - res.status(200).json(data); - }, - serverError(err) { - err.httpCode = 500; + if (_.isNil(data)) + return res.sendStatus(404); - res.api.error(err); + res.status(200).json(data); } }; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/middlewares/demo.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/middlewares/demo.js b/modules/web-console/backend/middlewares/demo.js new file mode 100644 index 0000000..537ede1 --- /dev/null +++ b/modules/web-console/backend/middlewares/demo.js @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +// Fire me up! + +module.exports = { + implements: 'middlewares:demo', + factory: () => { + return (req, res, next) => { + req.demo = () => req.header('IgniteDemoMode'); + + next(); + }; + } +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/migrations/1502249492000-invalidate_rename.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1502249492000-invalidate_rename.js b/modules/web-console/backend/migrations/1502249492000-invalidate_rename.js new file mode 100644 index 0000000..1fe7687 --- /dev/null +++ b/modules/web-console/backend/migrations/1502249492000-invalidate_rename.js @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +exports.up = function up(done) { + this('Cache').update({}, { $rename: { invalidate: 'isInvalidate' } }, { multi: true }) + .then(() => done()) + .catch(done); +}; + +exports.down = function down(done) { + this('Cache').update({}, { $rename: { isInvalidate: 'invalidate' } }, { multi: true }) + .then(() => done()) + .catch(done); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/migrations/1502432624000-cache-index.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1502432624000-cache-index.js b/modules/web-console/backend/migrations/1502432624000-cache-index.js new file mode 100644 index 0000000..147e2ad --- /dev/null +++ b/modules/web-console/backend/migrations/1502432624000-cache-index.js @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const recreateIndex = require('./migration-utils').recreateIndex; + +exports.up = function up(done) { + recreateIndex(done, this('Cache').collection, + 'name_1_space_1', + {name: 1, space: 1}, + {name: 1, space: 1, clusters: 1}); +}; + +exports.down = function down(done) { + recreateIndex(done, this('Cache').collection, + 'name_1_space_1_clusters_1', + {name: 1, space: 1, clusters: 1}, + {name: 1, space: 1}); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/migrations/1504672035000-igfs-index.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1504672035000-igfs-index.js b/modules/web-console/backend/migrations/1504672035000-igfs-index.js new file mode 100644 index 0000000..e802ca9 --- /dev/null +++ b/modules/web-console/backend/migrations/1504672035000-igfs-index.js @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const recreateIndex = require('./migration-utils').recreateIndex; + +exports.up = function up(done) { + recreateIndex(done, this('Igfs').collection, + 'name_1_space_1', + {name: 1, space: 1}, + {name: 1, space: 1, clusters: 1}); +}; + +exports.down = function down(done) { + recreateIndex(done, this('Igfs').collection, + 'name_1_space_1_clusters_1', + {name: 1, space: 1, clusters: 1}, + {name: 1, space: 1}); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/migrations/1505114649000-models-index.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1505114649000-models-index.js b/modules/web-console/backend/migrations/1505114649000-models-index.js new file mode 100644 index 0000000..c007b01 --- /dev/null +++ b/modules/web-console/backend/migrations/1505114649000-models-index.js @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const recreateIndex = require('./migration-utils').recreateIndex; + +exports.up = function up(done) { + recreateIndex(done, this('DomainModel').collection, + 'valueType_1_space_1', + {valueType: 1, space: 1}, + {valueType: 1, space: 1, clusters: 1}); +}; + +exports.down = function down(done) { + recreateIndex(done, this('DomainModel').collection, + 'valueType_1_space_1_clusters_1', + {valueType: 1, space: 1, clusters: 1}, + {valueType: 1, space: 1}); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/migrations/1508395969410-init-registered-date.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1508395969410-init-registered-date.js b/modules/web-console/backend/migrations/1508395969410-init-registered-date.js index b994cac..365f6ce 100644 --- a/modules/web-console/backend/migrations/1508395969410-init-registered-date.js +++ b/modules/web-console/backend/migrations/1508395969410-init-registered-date.js @@ -18,10 +18,12 @@ const _ = require('lodash'); exports.up = function up(done) { - const accounts = this('Account'); + const accountsModel = this('Account'); - accounts.find({}).lean().exec() - .then((data) => _.forEach(data, (acc) => accounts.update({_id: acc._id}, {$set: {registered: acc.lastLogin}}, {upsert: true}).exec())) + accountsModel.find({}).lean().exec() + .then((accounts) => _.reduce(accounts, (start, account) => start + .then(() => accountsModel.update({_id: account._id}, {$set: {registered: account.lastLogin}}).exec())), + Promise.resolve()) .then(() => done()) .catch(done); }; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/migrations/1516948939797-migrate-configs.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/1516948939797-migrate-configs.js b/modules/web-console/backend/migrations/1516948939797-migrate-configs.js new file mode 100644 index 0000000..8a37274 --- /dev/null +++ b/modules/web-console/backend/migrations/1516948939797-migrate-configs.js @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const _ = require('lodash'); + +const log = require('./migration-utils').log; + +const createClusterForMigration = require('./migration-utils').createClusterForMigration; +const getClusterForMigration = require('./migration-utils').getClusterForMigration; +const createCacheForMigration = require('./migration-utils').createCacheForMigration; +const getCacheForMigration = require('./migration-utils').getCacheForMigration; + +function linkCacheToCluster(clustersModel, cluster, cachesModel, cache, domainsModel) { + return clustersModel.update({_id: cluster._id}, {$addToSet: {caches: cache._id}}).exec() + .then(() => cachesModel.update({_id: cache._id}, {clusters: [cluster._id]}).exec()) + .then(() => { + if (_.isEmpty(cache.domains)) + return Promise.resolve(); + + return _.reduce(cache.domains, (start, domain) => start.then(() => { + return domainsModel.update({_id: domain}, {clusters: [cluster._id]}).exec() + .then(() => clustersModel.update({_id: cluster._id}, {$addToSet: {models: domain}}).exec()); + }), Promise.resolve()); + }) + .catch((err) => log(`Failed link cache to cluster [cache=${cache.name}, cluster=${cluster.name}, err=${err}]`)); +} + +function cloneCache(clustersModel, cachesModel, domainsModel, cache) { + const cacheId = cache._id; + const clusters = cache.clusters; + + delete cache._id; + cache.clusters = []; + + if (cache.cacheStoreFactory && cache.cacheStoreFactory.kind === null) + delete cache.cacheStoreFactory.kind; + + return _.reduce(clusters, (start, cluster, idx) => start.then(() => { + const newCache = _.clone(cache); + + newCache.clusters = [cluster]; + + if (idx > 0) { + return clustersModel.update({_id: {$in: newCache.clusters}}, {$pull: {caches: cacheId}}, {multi: true}).exec() + .then(() => cachesModel.create(newCache)) + .then((clone) => clustersModel.update({_id: {$in: newCache.clusters}}, {$addToSet: {caches: clone._id}}, {multi: true}).exec() + .then(() => clone)) + .then((clone) => { + const domainIds = newCache.domains; + + if (_.isEmpty(domainIds)) + return Promise.resolve(); + + return _.reduce(domainIds, (start, domainId) => start.then(() => { + return domainsModel.findOne({_id: domainId}).lean().exec() + .then((domain) => { + delete domain._id; + + const newDomain = _.clone(domain); + + newDomain.caches = [clone._id]; + newDomain.clusters = [cluster]; + + return domainsModel.create(newDomain) + .then((createdDomain) => clustersModel.update({_id: cluster}, {$addToSet: {models: createdDomain._id}}).exec()); + }) + .catch((err) => log(`Failed to duplicate domain model[domain=${domainId}], cache=${clone.name}, err=${err}`)); + }), Promise.resolve()); + }) + .catch((err) => log(`Failed to clone cache[id=${cacheId}, name=${cache.name}, err=${err}]`)); + } + + return cachesModel.update({_id: cacheId}, {clusters: [cluster]}).exec(); + }), Promise.resolve()); +} + +function migrateCache(clustersModel, cachesModel, domainsModel, cache) { + const len = _.size(cache.clusters); + + if (len < 1) { + log(`Found cache not linked to cluster [cache=${cache.name}]`); + + return getClusterForMigration(clustersModel) + .then((clusterLostFound) => linkCacheToCluster(clustersModel, clusterLostFound, cachesModel, cache, domainsModel)); + } + + if (len > 1) { + log(`Found cache linked to many clusters [cache=${cache.name}, cnt=${len}]`); + + return cloneCache(clustersModel, cachesModel, domainsModel, cache); + } + + // Nothing to migrate, cache linked to cluster 1-to-1. + return Promise.resolve(); +} + +function migrateCaches(clustersModel, cachesModel, domainsModel) { + log('Caches migration started...'); + + return cachesModel.find({}).lean().exec() + .then((caches) => { + log(`Caches to migrate: ${_.size(caches)}`); + + _.reduce(caches, (start, cache) => start.then(() => migrateCache(clustersModel, cachesModel, domainsModel, cache)), Promise.resolve()) + .then(() => log('Caches migration finished.')); + }) + .catch((err) => log('Caches migration failed: ' + err)); +} + +function linkIgfsToCluster(clustersModel, cluster, igfsModel, igfs) { + return clustersModel.update({_id: cluster._id}, {$addToSet: {igfss: igfs._id}}).exec() + .then(() => igfsModel.update({_id: igfs._id}, {clusters: [cluster._id]}).exec()) + .catch((err) => log(`Failed link IGFS to cluster [IGFS=${igfs.name}, cluster=${cluster.name}, err=${err}]`)); +} + +function cloneIgfs(clustersModel, igfsModel, igfs) { + const igfsId = igfs._id; + const clusters = igfs.clusters; + + delete igfs._id; + igfs.clusters = []; + + return _.reduce(clusters, (start, cluster, idx) => { + const newIgfs = _.clone(igfs); + + newIgfs.clusters = [cluster]; + + if (idx > 0) { + return clustersModel.update({_id: {$in: newIgfs.clusters}}, {$pull: {igfss: igfsId}}, {multi: true}).exec() + .then(() => igfsModel.create(newIgfs)) + .then((clone) => clustersModel.update({_id: {$in: igfs.newIgfs}}, {$addToSet: {igfss: clone._id}}, {multi: true}).exec()) + .catch((err) => log(`Failed to clone IGFS: id=${igfsId}, name=${igfs.name}, err=${err}`)); + } + + return igfsModel.update({_id: igfsId}, {clusters: [cluster]}).exec(); + }, Promise.resolve()); +} + +function migrateIgfs(clustersModel, igfsModel, igfs) { + const len = _.size(igfs.clusters); + + if (len < 1) { + log(`Found IGFS not linked to cluster [IGFS=${igfs.name}]`); + + return getClusterForMigration(clustersModel) + .then((clusterLostFound) => linkIgfsToCluster(clustersModel, clusterLostFound, igfsModel, igfs)); + } + + if (len > 1) { + log(`Found IGFS linked to many clusters [IGFS=${igfs.name}, cnt=${len}]`); + + return cloneIgfs(clustersModel, igfsModel, igfs); + } + + // Nothing to migrate, IGFS linked to cluster 1-to-1. + return Promise.resolve(); +} + +function migrateIgfss(clustersModel, igfsModel) { + log('IGFS migration started...'); + + return igfsModel.find({}).lean().exec() + .then((igfss) => { + log(`IGFS to migrate: ${_.size(igfss)}`); + + return _.reduce(igfss, (start, igfs) => start.then(() => migrateIgfs(clustersModel, igfsModel, igfs)), Promise.resolve()) + .then(() => log('IGFS migration finished.')); + }) + .catch((err) => log('IGFS migration failed: ' + err)); +} + +function linkDomainToCluster(clustersModel, cluster, domainsModel, domain) { + return clustersModel.update({_id: cluster._id}, {$addToSet: {models: domain._id}}).exec() + .then(() => domainsModel.update({_id: domain._id}, {clusters: [cluster._id]}).exec()) + .catch((err) => log(`Failed link domain model to cluster [domain=${domain._id}, cluster=${cluster.name}, err=${err}]`)); +} + +function linkDomainToCache(cachesModel, cache, domainsModel, domain) { + return cachesModel.update({_id: cache._id}, {$addToSet: {domains: domain._id}}).exec() + .then(() => domainsModel.update({_id: domain._id}, {caches: [cache._id]}).exec()) + .catch((err) => log(`Failed link domain model to cache[cache=${cache.name}, domain=${domain._id}, err=${err}]`)); +} + +function migrateDomain(clustersModel, cachesModel, domainsModel, domain) { + if (_.isEmpty(domain.caches)) { + log(`Found domain model not linked to cache [domain=${domain._id}]`); + + return getClusterForMigration(clustersModel) + .then((clusterLostFound) => linkDomainToCluster(clustersModel, clusterLostFound, domainsModel, domain)) + .then(() => getCacheForMigration(cachesModel)) + .then((cacheLostFound) => linkDomainToCache(cachesModel, cacheLostFound, domainsModel, domain)); + } + + if (_.isEmpty(domain.clusters)) { + return cachesModel.findOne({_id: {$in: domain.caches}}).lean().exec() + .then((cache) => { + const clusterId = cache.clusters[0]; + + return domainsModel.update({_id: domain._id}, {clusters: [clusterId]}).exec() + .then(() => clustersModel.update({_id: clusterId}, {models: [domain._id]}).exec()); + }); + } + + // Nothing to migrate, other domains will be migrated with caches. + return Promise.resolve(); +} + +function migrateDomains(clustersModel, cachesModel, domainsModel) { + log('Domain models migration started...'); + + return domainsModel.find({}).lean().exec() + .then((domains) => { + log(`Domain models to migrate: ${_.size(domains)}`); + + return _.reduce(domains, (start, domain) => migrateDomain(clustersModel, cachesModel, domainsModel, domain), Promise.resolve()) + .then(() => log('Domain models migration finished.')); + }) + .catch((err) => log('Domain models migration failed: ' + err)); +} + +function deduplicate(title, model, name) { + return model.find({}).lean().exec() + .then((items) => { + log(`Deduplication of ${title} started...`); + + let cnt = 0; + + return _.reduce(items, (start, item) => start.then(() => { + const data = item[name]; + + const dataSz = _.size(data); + + if (dataSz < 2) + return Promise.resolve(); + + const deduped = _.uniqWith(data, _.isEqual); + + if (dataSz !== _.size(deduped)) { + return model.updateOne({_id: item._id}, {$set: {[name]: deduped}}) + .then(() => cnt++); + } + + + return Promise.resolve(); + }), Promise.resolve()) + .then(() => log(`Deduplication of ${title} finished: ${cnt}.`)); + }); +} + +exports.up = function up(done) { + const clustersModel = this('Cluster'); + const cachesModel = this('Cache'); + const domainsModel = this('DomainModel'); + const igfsModel = this('Igfs'); + + deduplicate('Cluster caches', clustersModel, 'caches') + .then(() => deduplicate('Cluster IGFS', clustersModel, 'igfss')) + .then(() => deduplicate('Cache clusters', cachesModel, 'clusters')) + .then(() => deduplicate('Cache domains', cachesModel, 'domains')) + .then(() => deduplicate('IGFS clusters', igfsModel, 'clusters')) + .then(() => deduplicate('Domain model caches', domainsModel, 'caches')) + .then(() => createClusterForMigration(clustersModel, cachesModel)) + .then(() => createCacheForMigration(clustersModel, cachesModel, domainsModel)) + .then(() => migrateCaches(clustersModel, cachesModel, domainsModel)) + .then(() => migrateIgfss(clustersModel, igfsModel)) + .then(() => migrateDomains(clustersModel, cachesModel, domainsModel)) + .then(() => done()) + .catch(done); +}; + +exports.down = function down(done) { + log('Model migration can not be reverted'); + + done(); +}; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/migrations/migration-utils.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/migration-utils.js b/modules/web-console/backend/migrations/migration-utils.js new file mode 100644 index 0000000..46f53c3 --- /dev/null +++ b/modules/web-console/backend/migrations/migration-utils.js @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function log(msg) { + console.log(`[${new Date().toISOString()}] ${msg}`); +} + +function recreateIndex0(done, model, oldIdxName, oldIdx, newIdx) { + return model.indexExists(oldIdxName) + .then((exists) => { + if (exists) { + return model.dropIndex(oldIdx) + .then(() => model.createIndex(newIdx, {unique: true, background: false})); + } + }) + .then(() => done()) + .catch((err) => { + if (err.code === 12587) { + log(`Background operation in progress for: ${oldIdxName}, will retry in 3 seconds.`); + + setTimeout(() => recreateIndex0(done, model, oldIdxName, oldIdx, newIdx), 3000); + } + else { + log(`Failed to recreate index: ${err}`); + + done(); + } + }); +} + +function recreateIndex(done, model, oldIdxName, oldIdx, newIdx) { + setTimeout(() => recreateIndex0(done, model, oldIdxName, oldIdx, newIdx), 1000); +} + +const LOST_AND_FOUND = 'LOST_AND_FOUND'; + +function createClusterForMigration(clustersModel, cachesModel) { + return cachesModel.findOne({clusters: []}) + .then((cache) => { + if (cache) { + clustersModel.create({ + space: cache.space, + name: LOST_AND_FOUND, + connector: {noDelay: true}, + communication: {tcpNoDelay: true}, + igfss: [], + caches: [], + binaryConfiguration: { + compactFooter: true, + typeConfigurations: [] + }, + discovery: { + kind: 'Multicast', + Multicast: {addresses: ['127.0.0.1:47500..47510']}, + Vm: {addresses: ['127.0.0.1:47500..47510']} + } + }); + } + }); +} + +function getClusterForMigration(clustersModel) { + return clustersModel.findOne({name: LOST_AND_FOUND}).lean().exec(); +} + +function createCacheForMigration(clustersModel, cachesModel, domainsModels) { + return domainsModels.findOne({caches: []}) + .then((domain) => getClusterForMigration(clustersModel) + .then((cluster) => cachesModel.create({ + space: domain.space, + name: LOST_AND_FOUND, + clusters: [cluster._id], + domains: [], + cacheMode: 'PARTITIONED', + atomicityMode: 'ATOMIC', + readFromBackup: true, + copyOnRead: true, + readThrough: false, + writeThrough: false, + sqlFunctionClasses: [], + writeBehindCoalescing: true, + cacheStoreFactory: { + CacheHibernateBlobStoreFactory: {hibernateProperties: []}, + CacheJdbcBlobStoreFactory: {connectVia: 'DataSource'} + }, + nearConfiguration: {}, + evictionPolicy: {} + })) + ); +} + +function getCacheForMigration(cachesModel) { + return cachesModel.findOne({name: LOST_AND_FOUND}); +} + +module.exports = { + log, + recreateIndex, + createClusterForMigration, + getClusterForMigration, + createCacheForMigration, + getCacheForMigration +}; + + + + http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/migrations/recreate-index.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/migrations/recreate-index.js b/modules/web-console/backend/migrations/recreate-index.js deleted file mode 100644 index 328ed43..0000000 --- a/modules/web-console/backend/migrations/recreate-index.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -module.exports = function(done, model, oldIdxName, oldIdx, newIdx) { - model.indexExists(oldIdxName) - .then((exists) => { - if (exists) { - return model.dropIndex(oldIdx) - .then(() => model.createIndex(newIdx, {unique: true})); - } - }) - .then(() => done()) - .catch(done); -}; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/routes/caches.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/caches.js b/modules/web-console/backend/routes/caches.js index d7ed8b8..25f76a1 100644 --- a/modules/web-console/backend/routes/caches.js +++ b/modules/web-console/backend/routes/caches.js @@ -30,6 +30,18 @@ module.exports.factory = function(mongo, cachesService) { return new Promise((factoryResolve) => { const router = new express.Router(); + router.get('/:_id', (req, res) => { + cachesService.get(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.delete('/', (req, res) => { + cachesService.remove(req.body.ids) + .then(res.api.ok) + .catch(res.api.error); + }); + /** * Save cache. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/routes/clusters.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/clusters.js b/modules/web-console/backend/routes/clusters.js index 24334c2..ac7b25e 100644 --- a/modules/web-console/backend/routes/clusters.js +++ b/modules/web-console/backend/routes/clusters.js @@ -23,13 +23,55 @@ const express = require('express'); module.exports = { implements: 'routes/clusters', - inject: ['mongo', 'services/clusters'] + inject: ['mongo', 'services/clusters', 'services/caches', 'services/domains', 'services/igfss'] }; -module.exports.factory = function(mongo, clustersService) { +module.exports.factory = function(mongo, clustersService, cachesService, domainsService, igfssService) { return new Promise((factoryResolve) => { const router = new express.Router(); + router.get('/:_id/caches', (req, res) => { + cachesService.shortList(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.get('/:_id/models', (req, res) => { + domainsService.shortList(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.get('/:_id/igfss', (req, res) => { + igfssService.shortList(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.get('/:_id', (req, res) => { + clustersService.get(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.get('/', (req, res) => { + clustersService.shortList(req.currentUserId(), req.demo()) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.put('/basic', (req, res) => { + clustersService.upsertBasic(req.currentUserId(), req.demo(), req.body) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.put('/', (req, res) => { + clustersService.upsert(req.currentUserId(), req.demo(), req.body) + .then(res.api.ok) + .catch(res.api.error); + }); + /** * Save cluster. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/routes/configuration.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/configuration.js b/modules/web-console/backend/routes/configuration.js index d9bde75..9ec002b 100644 --- a/modules/web-console/backend/routes/configuration.js +++ b/modules/web-console/backend/routes/configuration.js @@ -29,11 +29,21 @@ module.exports = { module.exports.factory = function(mongo, configurationsService) { return new Promise((factoryResolve) => { const router = new express.Router(); + /** * Get all user configuration in current space. */ router.get('/list', (req, res) => { - configurationsService.list(req.currentUserId(), req.header('IgniteDemoMode')) + configurationsService.list(req.currentUserId(), req.demo()) + .then(res.api.ok) + .catch(res.api.error); + }); + + /** + * Get user configuration in current space. + */ + router.get('/:_id', (req, res) => { + configurationsService.get(req.currentUserId(), req.demo(), req.params._id) .then(res.api.ok) .catch(res.api.error); }); http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/routes/domains.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/domains.js b/modules/web-console/backend/routes/domains.js index caa9201..9360421 100644 --- a/modules/web-console/backend/routes/domains.js +++ b/modules/web-console/backend/routes/domains.js @@ -30,6 +30,12 @@ module.exports.factory = (mongo, domainsService) => { return new Promise((factoryResolve) => { const router = new express.Router(); + router.get('/:_id', (req, res) => { + domainsService.get(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + /** * Save domain model. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/routes/igfss.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/routes/igfss.js b/modules/web-console/backend/routes/igfss.js index b95f21f..c975249 100644 --- a/modules/web-console/backend/routes/igfss.js +++ b/modules/web-console/backend/routes/igfss.js @@ -30,6 +30,18 @@ module.exports.factory = function(mongo, igfssService) { return new Promise((factoryResolve) => { const router = new express.Router(); + router.get('/:_id', (req, res) => { + igfssService.get(req.currentUserId(), req.demo(), req.params._id) + .then(res.api.ok) + .catch(res.api.error); + }); + + router.delete('/', (req, res) => { + igfssService.remove(req.body.ids) + .then(res.api.ok) + .catch(res.api.error); + }); + /** * Save IGFS. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/services/caches.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/services/caches.js b/modules/web-console/backend/services/caches.js index 9cb65a1..4573587 100644 --- a/modules/web-console/backend/services/caches.js +++ b/modules/web-console/backend/services/caches.js @@ -28,11 +28,11 @@ module.exports = { /** * @param mongo - * @param {SpacesService} spaceService + * @param {SpacesService} spacesService * @param errors * @returns {CachesService} */ -module.exports.factory = (mongo, spaceService, errors) => { +module.exports.factory = (mongo, spacesService, errors) => { /** * Convert remove status operation to own presentation. * @@ -101,6 +101,55 @@ module.exports.factory = (mongo, spaceService, errors) => { * Service for manipulate Cache entities. */ class CachesService { + static shortList(userId, demo, clusterId) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.Cache.find({space: {$in: spaceIds}, clusters: clusterId }).select('name cacheMode atomicityMode backups').lean().exec()); + } + + static get(userId, demo, _id) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.Cache.findOne({space: {$in: spaceIds}, _id}).lean().exec()); + } + + static upsertBasic(cache) { + if (_.isNil(cache._id)) + return Promise.reject(new errors.IllegalArgumentException('Cache id can not be undefined or null')); + + const query = _.pick(cache, ['space', '_id']); + const newDoc = _.pick(cache, ['space', '_id', 'name', 'cacheMode', 'atomicityMode', 'backups']); + + return mongo.Cache.update(query, {$set: newDoc}, {upsert: true}).exec() + .catch((err) => { + if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) + throw new errors.DuplicateKeyException(`Cache with name: "${cache.name}" already exist.`); + + throw err; + }) + .then((updated) => { + if (updated.nModified === 0) + return mongo.Cache.update(query, {$set: cache}, {upsert: true}).exec(); + + return updated; + }); + } + + static upsert(cache) { + if (_.isNil(cache._id)) + return Promise.reject(new errors.IllegalArgumentException('Cache id can not be undefined or null')); + + const query = _.pick(cache, ['space', '_id']); + + return mongo.Cache.update(query, {$set: cache}, {upsert: true}).exec() + .then(() => mongo.DomainModel.update({_id: {$in: cache.domains}}, {$addToSet: {caches: cache._id}}, {multi: true}).exec()) + .then(() => mongo.DomainModel.update({_id: {$nin: cache.domains}}, {$pull: {caches: cache._id}}, {multi: true}).exec()) + .catch((err) => { + if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) + throw new errors.DuplicateKeyException(`Cache with name: "${cache.name}" already exist.`); + + throw err; + }); + } + /** * Create or update cache. * @@ -125,21 +174,26 @@ module.exports.factory = (mongo, spaceService, errors) => { } /** - * Remove cache. + * Remove caches. * - * @param {mongo.ObjectId|String} cacheId - The cache id for remove. + * @param {Array.<String>|String} ids - The cache ids for remove. * @returns {Promise.<{rowsAffected}>} - The number of affected rows. */ - static remove(cacheId) { - if (_.isNil(cacheId)) + static remove(ids) { + if (_.isNil(ids)) return Promise.reject(new errors.IllegalArgumentException('Cache id can not be undefined or null')); - return mongo.Cluster.update({caches: {$in: [cacheId]}}, {$pull: {caches: cacheId}}, {multi: true}).exec() - .then(() => mongo.Cluster.update({}, {$pull: {checkpointSpi: {kind: 'Cache', Cache: {cache: cacheId}}}}, {multi: true}).exec()) - // TODO WC-201 fix clenup of cache on deletion for cluster service configuration. + ids = _.castArray(ids); + + if (_.isEmpty(ids)) + return Promise.resolve({rowsAffected: 0}); + + return mongo.Cluster.update({caches: {$in: ids}}, {$pull: {caches: {$in: ids}}}, {multi: true}).exec() + .then(() => mongo.Cluster.update({}, {$pull: {checkpointSpi: {kind: 'Cache', Cache: {cache: {$in: ids}}}}}, {multi: true}).exec()) + // TODO WC-201 fix cleanup of cache on deletion for cluster service configuration. // .then(() => mongo.Cluster.update({'serviceConfigurations.cache': cacheId}, {$unset: {'serviceConfigurations.$.cache': ''}}, {multi: true}).exec()) - .then(() => mongo.DomainModel.update({caches: {$in: [cacheId]}}, {$pull: {caches: cacheId}}, {multi: true}).exec()) - .then(() => mongo.Cache.remove({_id: cacheId}).exec()) + .then(() => mongo.DomainModel.update({caches: {$in: ids}}, {$pull: {caches: {$in: ids}}}, {multi: true}).exec()) + .then(() => mongo.Cache.remove({_id: {$in: ids}}).exec()) .then(convertRemoveStatus); } @@ -151,7 +205,7 @@ module.exports.factory = (mongo, spaceService, errors) => { * @returns {Promise.<{rowsAffected}>} - The number of affected rows. */ static removeAll(userId, demo) { - return spaceService.spaceIds(userId, demo) + return spacesService.spaceIds(userId, demo) .then(removeAllBySpaces) .then(convertRemoveStatus); } http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/services/clusters.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/services/clusters.js b/modules/web-console/backend/services/clusters.js index 49e6f09..0cc2b9f 100644 --- a/modules/web-console/backend/services/clusters.js +++ b/modules/web-console/backend/services/clusters.js @@ -23,16 +23,19 @@ const _ = require('lodash'); module.exports = { implements: 'services/clusters', - inject: ['mongo', 'services/spaces', 'errors'] + inject: ['mongo', 'services/spaces', 'services/caches', 'services/domains', 'services/igfss', 'errors'] }; /** * @param mongo * @param {SpacesService} spacesService + * @param {CachesService} cachesService + * @param {DomainsService} modelsService + * @param {IgfssService} igfssService * @param errors * @returns {ClustersService} */ -module.exports.factory = (mongo, spacesService, errors) => { +module.exports.factory = (mongo, spacesService, cachesService, modelsService, igfssService, errors) => { /** * Convert remove status operation to own presentation. * @@ -71,17 +74,17 @@ module.exports.factory = (mongo, spacesService, errors) => { */ const create = (cluster) => { return mongo.Cluster.create(cluster) - .then((savedCluster) => - mongo.Cache.update({_id: {$in: savedCluster.caches}}, {$addToSet: {clusters: savedCluster._id}}, {multi: true}).exec() - .then(() => mongo.Igfs.update({_id: {$in: savedCluster.igfss}}, {$addToSet: {clusters: savedCluster._id}}, {multi: true}).exec()) - .then(() => savedCluster) - ) .catch((err) => { if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) - throw new errors.DuplicateKeyException('Cluster with name: "' + cluster.name + '" already exist.'); + throw new errors.DuplicateKeyException(`Cluster with name: "${cluster.name}" already exist.`); else throw err; - }); + }) + .then((savedCluster) => + mongo.Cache.update({_id: {$in: savedCluster.caches}}, {$addToSet: {clusters: savedCluster._id}}, {multi: true}).exec() + .then(() => mongo.Igfs.update({_id: {$in: savedCluster.igfss}}, {$addToSet: {clusters: savedCluster._id}}, {multi: true}).exec()) + .then(() => savedCluster) + ); }; /** @@ -97,6 +100,110 @@ module.exports.factory = (mongo, spacesService, errors) => { }; class ClustersService { + static shortList(userId, demo) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.Cluster.find({space: {$in: spaceIds}}).select('name discovery.kind caches models igfss').lean().exec()) + .then((clusters) => _.map(clusters, (cluster) => ({ + _id: cluster._id, + name: cluster.name, + discovery: cluster.discovery.kind, + cachesCount: _.size(cluster.caches), + modelsCount: _.size(cluster.models), + igfsCount: _.size(cluster.igfss) + }))); + } + + static get(userId, demo, _id) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.Cluster.findOne({space: {$in: spaceIds}, _id}).lean().exec()); + } + + static normalize(spaceId, cluster, ...models) { + cluster.space = spaceId; + + _.forEach(models, (model) => { + _.forEach(model, (item) => { + item.space = spaceId; + item.clusters = [cluster._id]; + }); + }); + } + + static removedInCluster(oldCluster, newCluster, field) { + return _.difference(_.invokeMap(_.get(oldCluster, field), 'toString'), _.get(newCluster, field)); + } + + static upsertBasic(userId, demo, {cluster, caches}) { + if (_.isNil(cluster._id)) + return Promise.reject(new errors.IllegalArgumentException('Cluster id can not be undefined or null')); + + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => { + this.normalize(_.head(spaceIds), cluster, caches); + + const query = _.pick(cluster, ['space', '_id']); + const basicCluster = _.pick(cluster, [ + 'space', + '_id', + 'name', + 'discovery', + 'caches', + 'memoryConfiguration.memoryPolicies', + 'dataStorageConfiguration.defaultDataRegionConfiguration.maxSize' + ]); + + return mongo.Cluster.findOneAndUpdate(query, {$set: basicCluster}, {projection: 'caches', upsert: true}).lean().exec() + .catch((err) => { + if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) + throw new errors.DuplicateKeyException(`Cluster with name: "${cluster.name}" already exist.`); + + throw err; + }) + .then((oldCluster) => { + if (oldCluster) { + const ids = this.removedInCluster(oldCluster, cluster, 'caches'); + + return cachesService.remove(ids); + } + + cluster.caches = _.map(caches, '_id'); + + return mongo.Cluster.update(query, {$set: cluster, new: true}, {upsert: true}).exec(); + }); + }) + .then(() => _.map(caches, cachesService.upsertBasic)) + .then(() => ({rowsAffected: 1})); + } + + static upsert(userId, demo, {cluster, caches, models, igfss}) { + if (_.isNil(cluster._id)) + return Promise.reject(new errors.IllegalArgumentException('Cluster id can not be undefined or null')); + + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => { + this.normalize(_.head(spaceIds), cluster, caches, models, igfss); + + const query = _.pick(cluster, ['space', '_id']); + + return mongo.Cluster.findOneAndUpdate(query, {$set: cluster}, {projection: {models: 1, caches: 1, igfss: 1}, upsert: true}).lean().exec() + .catch((err) => { + if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) + throw new errors.DuplicateKeyException(`Cluster with name: "${cluster.name}" already exist.`); + + throw err; + }) + .then((oldCluster) => { + const modelIds = this.removedInCluster(oldCluster, cluster, 'models'); + const cacheIds = this.removedInCluster(oldCluster, cluster, 'caches'); + const igfsIds = this.removedInCluster(oldCluster, cluster, 'igfss'); + + return Promise.all([modelsService.remove(modelIds), cachesService.remove(cacheIds), igfssService.remove(igfsIds)]); + }); + }) + .then(() => Promise.all(_.concat(_.map(models, modelsService.upsert), _.map(caches, cachesService.upsert), _.map(igfss, igfssService.upsert)))) + .then(() => ({rowsAffected: 1})); + } + /** * Create or update cluster. * @@ -121,19 +228,31 @@ module.exports.factory = (mongo, spacesService, errors) => { } /** - * Remove cluster. + * Remove clusters. * - * @param {mongo.ObjectId|String} clusterId - The cluster id for remove. + * @param {Array.<String>|String} ids - The cluster ids for remove. * @returns {Promise.<{rowsAffected}>} - The number of affected rows. */ - static remove(clusterId) { - if (_.isNil(clusterId)) + static remove(ids) { + if (_.isNil(ids)) return Promise.reject(new errors.IllegalArgumentException('Cluster id can not be undefined or null')); - return mongo.Cache.update({clusters: {$in: [clusterId]}}, {$pull: {clusters: clusterId}}, {multi: true}).exec() - .then(() => mongo.Igfs.update({clusters: {$in: [clusterId]}}, {$pull: {clusters: clusterId}}, {multi: true}).exec()) - .then(() => mongo.Cluster.remove({_id: clusterId}).exec()) - .then(convertRemoveStatus); + if (_.isEmpty(ids)) + return Promise.resolve({rowsAffected: 0}); + + ids = _.castArray(ids); + + return Promise.all(_.map(ids, (id) => { + return mongo.Cluster.findByIdAndRemove(id).exec() + .then((cluster) => { + return Promise.all([ + mongo.DomainModel.remove({_id: {$in: cluster.models}}).exec(), + mongo.Cache.remove({_id: {$in: cluster.caches}}).exec(), + mongo.Igfs.remove({_id: {$in: cluster.igfss}}).exec() + ]); + }); + })) + .then(() => ({rowsAffected: ids.length})); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/services/configurations.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/services/configurations.js b/modules/web-console/backend/services/configurations.js index 36d9932..da431c3 100644 --- a/modules/web-console/backend/services/configurations.js +++ b/modules/web-console/backend/services/configurations.js @@ -52,6 +52,18 @@ module.exports.factory = (mongo, spacesService, clustersService, cachesService, ])) .then(([clusters, domains, caches, igfss]) => ({clusters, domains, caches, igfss, spaces})); } + + static get(userId, demo, _id) { + return clustersService.get(userId, demo, _id) + .then((cluster) => + Promise.all([ + mongo.Cache.find({space: cluster.space, _id: {$in: cluster.caches}}).lean().exec(), + mongo.DomainModel.find({space: cluster.space, _id: {$in: cluster.models}}).lean().exec(), + mongo.Igfs.find({space: cluster.space, _id: {$in: cluster.igfss}}).lean().exec() + ]) + .then(([caches, models, igfss]) => ({cluster, caches, models, igfss})) + ); + } } return ConfigurationsService; http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/services/domains.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/services/domains.js b/modules/web-console/backend/services/domains.js index 986991d..0139413 100644 --- a/modules/web-console/backend/services/domains.js +++ b/modules/web-console/backend/services/domains.js @@ -149,6 +149,87 @@ module.exports.factory = (mongo, spacesService, cachesService, errors) => { }; class DomainsService { + static shortList(userId, demo, clusterId) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => { + spaceIds = _.map(spaceIds, (spaceId) => mongo.ObjectId(spaceId)); + + return mongo.DomainModel.aggregate([ + {$match: {space: {$in: spaceIds}, clusters: mongo.ObjectId(clusterId)}}, + {$project: { + keyType: 1, + valueType: 1, + queryMetadata: 1, + hasIndex: { + $or: [ + { + $and: [ + {$eq: ['$queryMetadata', 'Annotations']}, + { + $or: [ + {$eq: ['$generatePojo', false]}, + { + $and: [ + {$eq: ['$databaseSchema', '']}, + {$eq: ['$databaseTable', '']} + ] + } + ] + } + ] + }, + {$gt: [{$size: {$ifNull: ['$keyFields', []]}}, 0]} + ] + } + }} + ]).exec(); + }); + } + + static get(userId, demo, _id) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.DomainModel.findOne({space: {$in: spaceIds}, _id}).lean().exec()); + } + + static upsert(model) { + if (_.isNil(model._id)) + return Promise.reject(new errors.IllegalArgumentException('Model id can not be undefined or null')); + + const query = _.pick(model, ['space', '_id']); + + return mongo.DomainModel.update(query, {$set: model}, {upsert: true}).exec() + .then(() => mongo.Cache.update({_id: {$in: model.caches}}, {$addToSet: {domains: model._id}}, {multi: true}).exec()) + .then(() => mongo.Cache.update({_id: {$nin: model.caches}}, {$pull: {domains: model._id}}, {multi: true}).exec()) + .then(() => _updateCacheStore(model.cacheStoreChanges)) + .catch((err) => { + if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) + throw new errors.DuplicateKeyException(`Model with value type: "${model.valueType}" already exist.`); + + throw err; + }); + } + + /** + * Remove model. + * + * @param {mongo.ObjectId|String} ids - The model id for remove. + * @returns {Promise.<{rowsAffected}>} - The number of affected rows. + */ + static remove(ids) { + if (_.isNil(ids)) + return Promise.reject(new errors.IllegalArgumentException('Model id can not be undefined or null')); + + ids = _.castArray(ids); + + if (_.isEmpty(ids)) + return Promise.resolve({rowsAffected: 0}); + + return mongo.Cache.update({domains: {$in: ids}}, {$pull: {domains: ids}}, {multi: true}).exec() + .then(() => mongo.Cluster.update({models: {$in: ids}}, {$pull: {models: ids}}, {multi: true}).exec()) + .then(() => mongo.DomainModel.remove({_id: {$in: ids}}).exec()) + .then(convertRemoveStatus); + } + /** * Batch merging domains. * @@ -169,21 +250,6 @@ module.exports.factory = (mongo, spacesService, cachesService, errors) => { } /** - * Remove domain. - * - * @param {mongo.ObjectId|String} domainId - The domain id for remove. - * @returns {Promise.<{rowsAffected}>} - The number of affected rows. - */ - static remove(domainId) { - if (_.isNil(domainId)) - return Promise.reject(new errors.IllegalArgumentException('Domain id can not be undefined or null')); - - return mongo.Cache.update({domains: {$in: [domainId]}}, {$pull: {domains: domainId}}, {multi: true}).exec() - .then(() => mongo.DomainModel.remove({_id: domainId}).exec()) - .then(convertRemoveStatus); - } - - /** * Remove all domains by user. * @param {mongo.ObjectId|String} userId - The user id that own domain. * @param {Boolean} demo - The flag indicates that need lookup in demo space. http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/services/igfss.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/services/igfss.js b/modules/web-console/backend/services/igfss.js index 5296f16..b75d677 100644 --- a/modules/web-console/backend/services/igfss.js +++ b/modules/web-console/backend/services/igfss.js @@ -93,6 +93,31 @@ module.exports.factory = (mongo, spacesService, errors) => { }; class IgfssService { + static shortList(userId, demo, clusterId) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.Igfs.find({space: {$in: spaceIds}, clusters: clusterId }).select('name defaultMode affinnityGroupSize').lean().exec()); + } + + static get(userId, demo, _id) { + return spacesService.spaceIds(userId, demo) + .then((spaceIds) => mongo.Igfs.findOne({space: {$in: spaceIds}, _id}).lean().exec()); + } + + static upsert(igfs) { + if (_.isNil(igfs._id)) + return Promise.reject(new errors.IllegalArgumentException('IGFS id can not be undefined or null')); + + const query = _.pick(igfs, ['space', '_id']); + + return mongo.Igfs.update(query, {$set: igfs}, {upsert: true}).exec() + .catch((err) => { + if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR) + throw new errors.DuplicateKeyException(`IGFS with name: "${igfs.name}" already exist.`); + + throw err; + }); + } + /** * Create or update IGFS. * @@ -117,22 +142,27 @@ module.exports.factory = (mongo, spacesService, errors) => { } /** - * Remove IGFS. + * Remove IGFSs. * - * @param {mongo.ObjectId|String} igfsId - The IGFS id for remove. + * @param {Array.<String>|String} ids - The IGFS ids for remove. * @returns {Promise.<{rowsAffected}>} - The number of affected rows. */ - static remove(igfsId) { - if (_.isNil(igfsId)) + static remove(ids) { + if (_.isNil(ids)) return Promise.reject(new errors.IllegalArgumentException('IGFS id can not be undefined or null')); - return mongo.Cluster.update({igfss: {$in: [igfsId]}}, {$pull: {igfss: igfsId}}, {multi: true}).exec() - // TODO WC-201 fix clenup on node filter on deletion for cluster serviceConfigurations and caches. + ids = _.castArray(ids); + + if (_.isEmpty(ids)) + return Promise.resolve({rowsAffected: 0}); + + return mongo.Cluster.update({igfss: {$in: ids}}, {$pull: {igfss: {$in: ids}}}, {multi: true}).exec() + // TODO WC-201 fix cleanup on node filter on deletion for cluster serviceConfigurations and caches. // .then(() => mongo.Cluster.update({ 'serviceConfigurations.$.nodeFilter.kind': { $ne: 'IGFS' }, 'serviceConfigurations.nodeFilter.IGFS.igfs': igfsId}, // {$unset: {'serviceConfigurations.$.nodeFilter.IGFS.igfs': ''}}, {multi: true}).exec()) // .then(() => mongo.Cluster.update({ 'serviceConfigurations.nodeFilter.kind': 'IGFS', 'serviceConfigurations.nodeFilter.IGFS.igfs': igfsId}, // {$unset: {'serviceConfigurations.$.nodeFilter': ''}}, {multi: true}).exec()) - .then(() => mongo.Igfs.remove({_id: igfsId}).exec()) + .then(() => mongo.Igfs.remove({_id: {$in: ids}}).exec()) .then(convertRemoveStatus); } http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/services/sessions.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/services/sessions.js b/modules/web-console/backend/services/sessions.js index 0518ce2..0ea851b 100644 --- a/modules/web-console/backend/services/sessions.js +++ b/modules/web-console/backend/services/sessions.js @@ -51,7 +51,7 @@ module.exports.factory = (mongo, errors) => { return new Promise((resolve) => { delete session.viewedUser; - resolve(); + resolve(true); }); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/services/spaces.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/services/spaces.js b/modules/web-console/backend/services/spaces.js index 85f346e..fe62f77 100644 --- a/modules/web-console/backend/services/spaces.js +++ b/modules/web-console/backend/services/spaces.js @@ -57,7 +57,7 @@ module.exports.factory = (mongo, errors) => { */ static spaceIds(userId, demo) { return this.spaces(userId, demo) - .then((spaces) => spaces.map((space) => space._id)); + .then((spaces) => spaces.map((space) => space._id.toString())); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/f173be81/modules/web-console/backend/test/unit/CacheService.test.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/test/unit/CacheService.test.js b/modules/web-console/backend/test/unit/CacheService.test.js index 304f62c..52936a0 100644 --- a/modules/web-console/backend/test/unit/CacheService.test.js +++ b/modules/web-console/backend/test/unit/CacheService.test.js @@ -21,7 +21,7 @@ const testCaches = require('../data/caches.json'); const testAccounts = require('../data/accounts.json'); const testSpaces = require('../data/spaces.json'); -let cacheService; +let cachesService; let mongo; let errors; let db; @@ -34,7 +34,7 @@ suite('CacheServiceTestsSuite', () => { injector('dbHelper')]) .then(([_cacheService, _mongo, _errors, _db]) => { mongo = _mongo; - cacheService = _cacheService; + cachesService = _cacheService; errors = _errors; db = _db; }); @@ -42,12 +42,24 @@ suite('CacheServiceTestsSuite', () => { setup(() => db.init()); + test('Get cache', (done) => { + const _id = testCaches[0]._id; + + cachesService.get(testCaches[0].space, false, _id) + .then((cache) => { + assert.isNotNull(cache); + assert.equal(cache._id, _id); + }) + .then(done) + .catch(done); + }); + test('Create new cache', (done) => { const dupleCache = Object.assign({}, testCaches[0], {name: 'Other name'}); delete dupleCache._id; - cacheService.merge(dupleCache) + cachesService.merge(dupleCache) .then((cache) => mongo.Cache.findById(cache._id)) .then((cache) => assert.isNotNull(cache)) .then(done) @@ -59,7 +71,7 @@ suite('CacheServiceTestsSuite', () => { const cacheBeforeMerge = Object.assign({}, testCaches[0], {name: newName}); - cacheService.merge(cacheBeforeMerge) + cachesService.merge(cacheBeforeMerge) .then((cache) => mongo.Cache.findById(cache._id)) .then((cacheAfterMerge) => assert.equal(cacheAfterMerge.name, newName)) .then(done) @@ -71,7 +83,7 @@ suite('CacheServiceTestsSuite', () => { delete dupleCache._id; - cacheService.merge(dupleCache) + cachesService.merge(dupleCache) .catch((err) => { assert.instanceOf(err, errors.DuplicateKeyException); @@ -80,7 +92,7 @@ suite('CacheServiceTestsSuite', () => { }); test('Remove existed cache', (done) => { - cacheService.remove(testCaches[0]._id) + cachesService.remove(testCaches[0]._id) .then(({rowsAffected}) => assert.equal(rowsAffected, 1) ) @@ -93,7 +105,7 @@ suite('CacheServiceTestsSuite', () => { }); test('Remove cache without identifier', (done) => { - cacheService.remove() + cachesService.remove() .catch((err) => { assert.instanceOf(err, errors.IllegalArgumentException); @@ -104,7 +116,7 @@ suite('CacheServiceTestsSuite', () => { test('Remove missed cache', (done) => { const validNoExistingId = 'FFFFFFFFFFFFFFFFFFFFFFFF'; - cacheService.remove(validNoExistingId) + cachesService.remove(validNoExistingId) .then(({rowsAffected}) => assert.equal(rowsAffected, 0) ) @@ -113,7 +125,7 @@ suite('CacheServiceTestsSuite', () => { }); test('Get all caches by space', (done) => { - cacheService.listBySpaces(testSpaces[0]._id) + cachesService.listBySpaces(testSpaces[0]._id) .then((caches) => assert.equal(caches.length, 5) ) @@ -122,7 +134,7 @@ suite('CacheServiceTestsSuite', () => { }); test('Remove all caches in space', (done) => { - cacheService.removeAll(testAccounts[0]._id, false) + cachesService.removeAll(testAccounts[0]._id, false) .then(({rowsAffected}) => assert.equal(rowsAffected, 5) ) @@ -130,6 +142,19 @@ suite('CacheServiceTestsSuite', () => { .catch(done); }); + test('List of all caches in cluster', (done) => { + cachesService.shortList(testAccounts[0]._id, false, testCaches[0].clusters[0]) + .then((caches) => { + assert.equal(caches.length, 2); + assert.isNotNull(caches[0]._id); + assert.isNotNull(caches[0].name); + assert.isNotNull(caches[0].cacheMode); + assert.isNotNull(caches[0].atomicityMode); + }) + .then(done) + .catch(done); + }); + test('Update linked entities on update cache', (done) => { // TODO IGNITE-3262 Add test. done();
