AMBARI-11320. Issues with Hive/Oozie database properties and Blueprint cluster install (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c0478e89 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c0478e89 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c0478e89 Branch: refs/heads/trunk Commit: c0478e89232cf6500c40b321418d55d195807d61 Parents: 162c10e Author: Alex Antonenko <[email protected]> Authored: Fri May 22 01:14:44 2015 +0300 Committer: Alex Antonenko <[email protected]> Committed: Fri May 22 11:45:57 2015 +0300 ---------------------------------------------------------------------- ambari-web/app/assets/test/tests.js | 1 + .../controllers/main/service/info/configs.js | 15 +- ambari-web/app/data/HDP2/site_properties.js | 9 + ambari-web/app/utils/configs/database.js | 190 +++++++++++++++++++ ambari-web/test/utils/configs/database_test.js | 132 +++++++++++++ 5 files changed, 334 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c0478e89/ambari-web/app/assets/test/tests.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/assets/test/tests.js b/ambari-web/app/assets/test/tests.js index 9dc1c12..e364b45 100644 --- a/ambari-web/app/assets/test/tests.js +++ b/ambari-web/app/assets/test/tests.js @@ -168,6 +168,7 @@ var files = ['test/init_model_test', 'test/utils/object_utils_test', 'test/utils/ui_effects_test', 'test/utils/updater_test', + 'test/utils/configs/database_test', 'test/utils/configs/config_property_helper_test', 'test/utils/configs/modification_handlers/modification_handler_test', 'test/views/common/chart/linear_time_test', http://git-wip-us.apache.org/repos/asf/ambari/blob/c0478e89/ambari-web/app/controllers/main/service/info/configs.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/info/configs.js b/ambari-web/app/controllers/main/service/info/configs.js index f492da1..7207e1c 100644 --- a/ambari-web/app/controllers/main/service/info/configs.js +++ b/ambari-web/app/controllers/main/service/info/configs.js @@ -19,6 +19,7 @@ var App = require('app'); require('controllers/wizard/slave_component_groups_controller'); var batchUtils = require('utils/batch_scheduled_requests'); +var databaseUtils = require('utils/configs/database'); App.MainServiceInfoConfigsController = Em.Controller.extend(App.ServerValidatorMixin, App.EnhancedConfigsMixin, App.PreloadRequestsChainMixin, App.ThemesMappingMixin, App.VersionsMappingMixin, App.ConfigsSaverMixin, { @@ -823,6 +824,7 @@ App.MainServiceInfoConfigsController = Em.Controller.extend(App.ServerValidatorM var serviceConfig = App.config.createServiceConfig(serviceName); //Make SecondaryNameNode invisible on enabling namenode HA var configsByService = this.get('allConfigs').filterProperty('serviceName', serviceName); + databaseUtils.bootstrapDatabaseProperties(configsByService, serviceName); this.loadConfigs(configsByService, serviceConfig); if (serviceConfig.get('serviceName') === 'HDFS') { if (App.get('isHaEnabled')) { @@ -1207,19 +1209,6 @@ App.MainServiceInfoConfigsController = Em.Controller.extend(App.ServerValidatorM hostProperties.forEach(function (h) { this.setHostForService(h.serviceName, h.componentName, h.hostProperty, h.m); }, this); - - if (serviceName === 'HIVE') { - var hiveDb = configs.findProperty('name', 'hive_database').value; - if (['Existing MySQL Database', 'Existing Oracle Database', 'Existing PostgreSQL Database', 'Existing MSSQL Server database with SQL authentication', 'Existing MSSQL Server database with integrated authentication'].contains(hiveDb)) { - configs.findProperty('name', 'hive_hostname').isVisible = true; - } - } - if (serviceName === 'OOZIE') { - var oozieDb = configs.findProperty('name', 'oozie_database').value; - if (['Existing MySQL Database', 'Existing Oracle Database', 'Existing PostgreSQL Database', 'Existing MSSQL Server database with SQL authentication', 'Existing MSSQL Server database with integrated authentication'].contains(oozieDb)) { - configs.findProperty('name', 'oozie_hostname').isVisible = true; - } - } }, /** http://git-wip-us.apache.org/repos/asf/ambari/blob/c0478e89/ambari-web/app/data/HDP2/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js index ba25035..1ea26a5 100644 --- a/ambari-web/app/data/HDP2/site_properties.js +++ b/ambari-web/app/data/HDP2/site_properties.js @@ -3443,6 +3443,7 @@ var hdp2properties = [ "isReconfigurable": true, "displayType": "host", "isOverridable": false, + "isRequiredByAgent": false, "isVisible": false, "isObserved": true, "serviceName": "HIVE", @@ -3458,6 +3459,7 @@ var hdp2properties = [ "recommendedValue": "", "displayType": "host", "isOverridable": false, + "isRequiredByAgent": false, "isVisible": false, "isObserved": true, "serviceName": "HIVE", @@ -3473,6 +3475,7 @@ var hdp2properties = [ "recommendedValue": "", "displayType": "host", "isOverridable": false, + "isRequiredByAgent": false, "isVisible": false, "isObserved": true, "serviceName": "HIVE", @@ -3488,6 +3491,7 @@ var hdp2properties = [ "recommendedValue": "", "displayType": "host", "isOverridable": false, + "isRequiredByAgent": false, "isVisible": false, "isObserved": true, "serviceName": "HIVE", @@ -3520,6 +3524,7 @@ var hdp2properties = [ "recommendedValue": "", "displayType": "host", "isOverridable": false, + "isRequiredByAgent": false, "isVisible": false, "isObserved": true, "serviceName": "HIVE", @@ -3535,6 +3540,7 @@ var hdp2properties = [ "recommendedValue": "", "displayType": "host", "isOverridable": false, + "isRequiredByAgent": false, "isVisible": false, "isObserved": true, "serviceName": "HIVE", @@ -3898,6 +3904,7 @@ var hdp2properties = [ "isOverridable": false, "displayType": "host", "isVisible": false, + "isRequiredByAgent": false, "isObserved": true, "serviceName": "OOZIE", "filename": "oozie-env.xml", @@ -3913,6 +3920,7 @@ var hdp2properties = [ "isOverridable": false, "displayType": "host", "isVisible": false, + "isRequiredByAgent": false, "isObserved": true, "serviceName": "OOZIE", "filename": "oozie-env.xml", @@ -3928,6 +3936,7 @@ var hdp2properties = [ "isOverridable": false, "displayType": "host", "isVisible": false, + "isRequiredByAgent": false, "isObserved": true, "serviceName": "OOZIE", "filename": "oozie-env.xml", http://git-wip-us.apache.org/repos/asf/ambari/blob/c0478e89/ambari-web/app/utils/configs/database.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/database.js b/ambari-web/app/utils/configs/database.js new file mode 100644 index 0000000..2ee3f65 --- /dev/null +++ b/ambari-web/app/utils/configs/database.js @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var validators = require('utils/validator'); +/** + * Helper methods to process database values and properties + * @module utils/configs/database + */ +module.exports = { + /** + * Database type properties with <code>options</code> attribute usually displayed as radio-button. + * <code>options</code> attribute contains mapping of database type to special properties name such as + * database host and database name. + * @type {object} + */ + dbServicePropertyMap: { + HIVE: { + dbType: 'hive_database', + databaseName: 'ambari.hive.db.schema.name', + connectionUrl: 'javax.jdo.option.ConnectionURL', + fallbackHostName: 'hive_ambari_host' + }, + OOZIE: { + dbType: 'oozie_database', + connectionUrl: 'oozie.service.JPAService.jdbc.url', + databaseName: 'oozie.db.schema.name', + fallbackHostName: 'oozieserver_host' + } + }, + + /** + * Map of jdbc url patterns related to supported database types. + * + * @type {object} + */ + DB_JDBC_PATTERNS: { + mysql: 'jdbc:mysql://{0}/{1}', + mssql: 'jdbc:sqlserver://{0};databaseName={1}', + postgres: 'jdbc:postgresql://{0}:5432/{1}', + derby: 'jdbc:derby:{0}/{1}', + oracle: 'jdbc:oracle:thin:@//{0}:1521/{1}' + }, + + /** + * Setup database related properties. + * + * @method bootstrapDatabaseProperties + * @param {App.ServiceConfigProperty[]} serviceConfigs + * @param {string} [serviceName=false] + */ + bootstrapDatabaseProperties: function(serviceConfigs, serviceName) { + var self = this; + var supportedServices = Em.keys(this.dbServicePropertyMap); + if (serviceName && !supportedServices.contains(serviceName)) return; + var serviceNames = serviceName ? [serviceName] : serviceConfigs.mapProperty('serviceName').uniq(); + serviceNames.forEach(function(serviceName) { + if (!supportedServices.contains(serviceName)) return; + var configs = serviceConfigs.filterProperty('serviceName', serviceName) || []; + var connectionConfigs = self.dbServicePropertyMap[serviceName]; + var databaseTypeProperty = configs.findProperty('name', connectionConfigs.dbType); + if (!databaseTypeProperty) return; + var databaseTypePropertyIndex = configs.indexOf(databaseTypeProperty); + var generatedProperties = self.getPropsByOptions(databaseTypeProperty, configs); + var jdbcObject = self.parseJdbcUrl(Em.get(configs.findProperty('name', connectionConfigs.connectionUrl), 'value')); + generatedProperties.forEach(function(property) { + if (Em.get(property, 'name').endsWith('_host')) { + // set UI host names for each database type with value parsed from jdbc connection url + // if value is not ip or hostname (in case of New Derby Database) for Oozie set <code>fallbackUrl</code> + // from <code>dbServicePropertyMap</code> + var dbHostName; + if (self.isValidHostname(jdbcObject.location)) { + dbHostName = jdbcObject.location; + } else { + dbHostName = Em.get(configs.findProperty('name', self.dbServicePropertyMap[serviceName].fallbackHostName), 'recommendedValue'); + dbHostName = Em.isArray(dbHostName) ? dbHostName[0] : dbHostName; + } + Em.setProperties(property, { + value: dbHostName + }); + self.addPropertyToServiceConfigs(property, serviceConfigs, databaseTypePropertyIndex); + } + }); + }); + }, + + isValidHostname: function(value) { + return validators.isHostname(value) || validators.isIpAddress(value); + }, + + /** + * Add UI specific property to serviceConfigObject if it does not exist, and update value for existed property. + * This code affects properties related to `_host` and `_database` which are hardcoded on UI. + * + * @param {object} property - property to append/update + * @param {App.ServiceConfigProperty[]} configs - loaded and processed service configs + * @param {integer} index of first occurrence of database type in config + */ + addPropertyToServiceConfigs: function(property, configs, index) { + var configProperty = configs.findProperty('name', Em.get(property, 'name')); + if (configProperty) { + Em.set(configProperty, 'value', Em.get(property, 'value')); + } else { + if (index) { + configs.insertAt(index, App.ServiceConfigProperty.create(property)); + } else { + configs.pushObject(App.ServiceConfigProperty.create(property)); + } + } + }, + + /** + * Get hardcoded properties from site_properties.js which UI use to display host name and database name. + * + * @method getPropsByOptions + * @param {object} databaseTypeProperty - hardcoded property from site_properties.js usualy used as radiobutton + * @param {App.ServiceConfigProperty[]} configs - loaded and processed configs + * @returns {object[]} - hardcoded properties from site_properties.js related to database name and location + */ + getPropsByOptions: function(databaseTypeProperty) { + Em.assert('Property related to database type should contains `options` attribute', databaseTypeProperty.get('options')); + return databaseTypeProperty.options.mapProperty('foreignKeys').reduce(function(p,c) { + return p.concat(c); + }).uniq().map(function(name) { + return App.config.get('preDefinedSiteProperties').findProperty('name', name) || null; + }).compact(); + }, + + /** + * Get database location from jdbc url value. + * + * @method getDBLocationFromJDBC + * @param {string} jdbcUrl - url to parse + * @returns {string|null} + */ + getDBLocationFromJDBC: function(jdbcUrl) { + var self = this; + var matches = Em.keys(this.DB_JDBC_PATTERNS).map(function(key) { + var reg = new RegExp(self.DB_JDBC_PATTERNS[key].format('(.*)', '(.*)')); + return jdbcUrl.match(reg); + }).compact(); + if (matches.length) { + var dbLocation = Em.get(matches, '0.1'); + if (dbLocation.startsWith('${')) { + return Em.getWithDefault(matches, '0.0', '').match(/\${[^}]+}/)[0]; + } + return dbLocation != '{0}' ? dbLocation : null; + } else { + return null; + } + }, + + parseJdbcUrl: function(jdbcUrl) { + var self = this; + var result = { + dbType: null, + location: null, + databaseName: null + }; + var dbName; + + result.dbType = Em.keys(this.DB_JDBC_PATTERNS).filter(function(key) { + var scheme = self.DB_JDBC_PATTERNS[key].match(/^jdbc:(\w+):/)[1]; + return new RegExp('jdbc:' + scheme).test(jdbcUrl); + })[0]; + + result.location = this.getDBLocationFromJDBC(jdbcUrl); + if (!jdbcUrl.endsWith('{1}')) { + dbName = jdbcUrl.replace(this.DB_JDBC_PATTERNS[result.dbType].format(result.location,''), ''); + if (dbName) { + result.databaseName = dbName.split(/[;|?]/)[0]; + } + } + return result; + } +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/c0478e89/ambari-web/test/utils/configs/database_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/utils/configs/database_test.js b/ambari-web/test/utils/configs/database_test.js new file mode 100644 index 0000000..de95750 --- /dev/null +++ b/ambari-web/test/utils/configs/database_test.js @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var App = require('app'); +var dbUtils = require('utils/configs/database'); + +describe('Database Utils', function() { + describe('#getDBLocationFromJDBC', function() { + [ + { + jdbcUrl: 'jdbc:mysql://localhost/somedb', + e: 'localhost' + }, + { + jdbcUrl: 'jdbc:postgresql://some.hostname.com:5432/somedb', + e: 'some.hostname.com' + }, + { + jdbcUrl: 'jdbc:derby:/some/dir/another_dir/somedb', + e: '/some/dir/another_dir' + }, + { + jdbcUrl: 'jdbc:derby:${oozie-env/data-dir}/${oozie-env/database_name}-db', + e: '${oozie-env/data-dir}' + }, + { + jdbcUrl: 'jdbc:sqlserver://127.0.0.1;databaseName=some-db;integratedSecurity=true', + e: '127.0.0.1' + }, + { + jdbcUrl: 'jdbc:oracle:thin:@//localhost.com:1521/someDb', + e: 'localhost.com' + }, + { + jdbcUrl: 'jdbc:oracle:thin:@//{0}:1521/{1}', + e: null + } + ].forEach(function(test) { + it('when jdbc url is ' + test.jdbcUrl + ' host name is ' + test.e, function() { + expect(dbUtils.getDBLocationFromJDBC(test.jdbcUrl)).to.eql(test.e); + }); + }); + }); + + describe('#parseJdbcUrl', function() { + [ + { + jdbcUrl: 'jdbc:mysql://localhost/somedb', + e: { + dbType: 'mysql', + location: 'localhost', + databaseName: 'somedb' + } + }, + { + jdbcUrl: 'jdbc:postgresql://some.hostname.com:5432/somedb', + e: { + dbType: 'postgres', + location: 'some.hostname.com', + databaseName: 'somedb' + } + }, + { + jdbcUrl: 'jdbc:derby:/some/dir/another_dir/somedb', + e: { + dbType: 'derby', + location: '/some/dir/another_dir', + databaseName: 'somedb' + } + }, + { + jdbcUrl: 'jdbc:derby:${oozie-env/data-dir}/${oozie-env/database_name}-db', + e: { + dbType: 'derby', + location: '${oozie-env/data-dir}', + databaseName: '${oozie-env/database_name}-db' + } + }, + { + jdbcUrl: 'jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true', + e: { + dbType: 'derby', + location: '${oozie.data.dir}', + databaseName: '${oozie.db.schema.name}-db' + } + }, + { + jdbcUrl: 'jdbc:sqlserver://127.0.0.1;databaseName=some-db;integratedSecurity=true', + e: { + dbType: 'mssql', + location: '127.0.0.1', + databaseName: 'some-db' + } + }, + { + jdbcUrl: 'jdbc:oracle:thin:@//localhost.com:1521/someDb', + e: { + dbType: 'oracle', + location: 'localhost.com', + databaseName: 'someDb' + } + }, + { + jdbcUrl: 'jdbc:oracle:thin:@//{0}:1521/{1}', + e: { + dbType: 'oracle', + location: null, + databaseName: null + } + } + ].forEach(function(test) { + it('when jdbc url is ' + test.jdbcUrl + ' result is ' + JSON.stringify(test.e), function() { + expect(dbUtils.parseJdbcUrl(test.jdbcUrl)).to.be.eql(test.e); + }); + }); + }); +});
