AMBARI-14518 hadoop.proxyuser.yarn.hosts does not include both RM's in HA deployment. (ababiichuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6c71cd1f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6c71cd1f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6c71cd1f Branch: refs/heads/branch-dev-patch-upgrade Commit: 6c71cd1f022841d94152b6c704a17374bb4ba4c2 Parents: 48c0df4 Author: ababiichuk <[email protected]> Authored: Tue Dec 29 15:36:18 2015 +0200 Committer: ababiichuk <[email protected]> Committed: Tue Dec 29 17:48:58 2015 +0200 ---------------------------------------------------------------------- .../resourceManager/step3_controller.js | 12 +++++- .../resourceManager/step4_controller.js | 40 ++++++++++---------- ambari-web/app/data/HDP2/rm_ha_properties.js | 20 +++++++++- ambari-web/app/messages.js | 3 +- ...onfig_with_override_recommendation_parser.js | 1 - .../utils/configs/rm_ha_config_initializer.js | 20 +++++++++- .../resourceManager/step3_controller_test.js | 24 +++++++++++- 7 files changed, 91 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6c71cd1f/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step3_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step3_controller.js b/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step3_controller.js index 1350811..d2b7a86 100644 --- a/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step3_controller.js +++ b/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step3_controller.js @@ -79,7 +79,8 @@ App.RMHighAvailabilityWizardStep3Controller = Em.Controller.extend({ loadConfigTagsSuccessCallback: function (data, opt, params) { var urlParams = '(type=zoo.cfg&tag=' + data.Clusters.desired_configs['zoo.cfg'].tag + ')|' + - '(type=yarn-site&tag=' + data.Clusters.desired_configs['yarn-site'].tag + ')'; + '(type=yarn-site&tag=' + data.Clusters.desired_configs['yarn-site'].tag + ')|' + + '(type=yarn-env&tag=' + data.Clusters.desired_configs['yarn-env'].tag + ')'; App.ajax.send({ name: 'reassign.load_configs', sender: this, @@ -134,11 +135,20 @@ App.RMHighAvailabilityWizardStep3Controller = Em.Controller.extend({ */ setDynamicConfigValues: function (configs, data) { var topologyLocalDB = this.get('content').getProperties(['masterComponentHosts', 'slaveComponentHosts', 'hosts']); + var yarnUser = data.items.findProperty('type', 'yarn-env').properties.yarn_user; + App.RmHaConfigInitializer.setup({ + yarnUser: yarnUser + }); var dependencies = this._prepareDependencies(data); + /** add dynamic property 'hadoop.proxyuser.' + yarnUser + '.hosts' **/ + var proxyUserConfig = App.ServiceConfigProperty.create(App.config.createDefaultConfig('hadoop.proxyuser.' + yarnUser + '.hosts', + 'MISC', 'core-site', false, {category : "HDFS", isUserProperty: false, isEditable: false, isOverridable: false})); + configs.configs.pushObject(proxyUserConfig); configs.configs.forEach(function (config) { App.RmHaConfigInitializer.initialValue(config, topologyLocalDB, dependencies); }); + App.RmHaConfigInitializer.cleanup(); return configs; }, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c71cd1f/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step4_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step4_controller.js b/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step4_controller.js index 2617721..2d4831a 100644 --- a/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step4_controller.js +++ b/ambari-web/app/controllers/main/admin/highAvailability/resourceManager/step4_controller.js @@ -26,7 +26,7 @@ App.RMHighAvailabilityWizardStep4Controller = App.HighAvailabilityProgressPageCo clusterDeployState: 'RM_HIGH_AVAILABILITY_DEPLOY', - commands: ['stopRequiredServices', 'installResourceManager', 'reconfigureYARN', 'reconfigureHAWQ', 'startAllServices'], + commands: ['stopRequiredServices', 'installResourceManager', 'reconfigureYARN', 'reconfigureHAWQ', 'reconfigureHDFS', 'startAllServices'], tasksMessagesPrefix: 'admin.rm_highAvailability.wizard.step', @@ -55,8 +55,12 @@ App.RMHighAvailabilityWizardStep4Controller = App.HighAvailabilityProgressPageCo this.loadConfigsTags("Hawq"); }, + reconfigureHDFS: function () { + this.loadConfigsTags("Hdfs"); + }, + loadConfigsTags: function (service) { - onLoadServiceConfigsTags = 'onLoad' + service + "ConfigsTags" + var onLoadServiceConfigsTags = 'onLoad' + service + "ConfigsTags"; App.ajax.send({ name: 'config.tags', sender: this, @@ -70,9 +74,10 @@ App.RMHighAvailabilityWizardStep4Controller = App.HighAvailabilityProgressPageCo name: 'reassign.load_configs', sender: this, data: { - urlParams: '(type=yarn-site&tag=' + data.Clusters.desired_configs['yarn-site'].tag + ')' + urlParams: '(type=yarn-site&tag=' + data.Clusters.desired_configs['yarn-site'].tag + ')', + type: 'yarn-site' }, - success: 'onLoadYarnConfigs', + success: 'onLoadConfigs', error: 'onTaskError' }); }, @@ -82,39 +87,34 @@ App.RMHighAvailabilityWizardStep4Controller = App.HighAvailabilityProgressPageCo name: 'reassign.load_configs', sender: this, data: { - urlParams: '(type=yarn-client&tag=' + data.Clusters.desired_configs['yarn-client'].tag + ')' + urlParams: '(type=yarn-client&tag=' + data.Clusters.desired_configs['yarn-client'].tag + ')', + type: 'yarn-client' }, - success: 'onLoadHawqConfigs', + success: 'onLoadConfigs', error: 'onTaskError' }); }, - onLoadYarnConfigs: function (data) { - var propertiesToAdd = this.get('content.configs').filterProperty('filename', 'yarn-site'); - propertiesToAdd.forEach(function (property) { - data.items[0].properties[property.name] = property.value; - }); - - var configData = this.reconfigureSites(['yarn-site'], data, Em.I18n.t('admin.highAvailability.step4.save.configuration.note').format(App.format.role('RESOURCEMANAGER'))); - + onLoadHdfsConfigsTags: function (data) { App.ajax.send({ - name: 'common.service.configurations', + name: 'reassign.load_configs', sender: this, data: { - desired_config: configData + urlParams: '(type=core-site&tag=' + data.Clusters.desired_configs['core-site'].tag + ')', + type: 'core-site' }, - success: 'onSaveConfigs', + success: 'onLoadConfigs', error: 'onTaskError' }); }, - onLoadHawqConfigs: function (data) { - var propertiesToAdd = this.get('content.configs').filterProperty('filename', 'yarn-client'); + onLoadConfigs: function (data, opt, params) { + var propertiesToAdd = this.get('content.configs').filterProperty('filename', params.type); propertiesToAdd.forEach(function (property) { data.items[0].properties[property.name] = property.value; }); - var configData = this.reconfigureSites(['yarn-client'], data, Em.I18n.t('admin.highAvailability.step4.save.configuration.note').format(App.format.role('RESOURCEMANAGER'))); + var configData = this.reconfigureSites([params.type], data, Em.I18n.t('admin.highAvailability.step4.save.configuration.note').format(App.format.role('RESOURCEMANAGER'))); App.ajax.send({ name: 'common.service.configurations', http://git-wip-us.apache.org/repos/asf/ambari/blob/6c71cd1f/ambari-web/app/data/HDP2/rm_ha_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/rm_ha_properties.js b/ambari-web/app/data/HDP2/rm_ha_properties.js index 5c85297..fb1bd13 100644 --- a/ambari-web/app/data/HDP2/rm_ha_properties.js +++ b/ambari-web/app/data/HDP2/rm_ha_properties.js @@ -23,9 +23,10 @@ module.exports = displayName: 'MISC', configCategories: [ App.ServiceConfigCategory.create({ name: 'YARN', displayName: 'YARN'}), - App.ServiceConfigCategory.create({ name: 'HAWQ', displayName: 'HAWQ'}) + App.ServiceConfigCategory.create({ name: 'HAWQ', displayName: 'HAWQ'}), + App.ServiceConfigCategory.create({ name: 'HDFS', displayName: 'HDFS'}) ], - sites: ['yarn-site', 'hawq-site'], + sites: ['yarn-site', 'hawq-site', 'core-site'], configs: [ /**********************************************HDFS***************************************/ { @@ -33,6 +34,7 @@ module.exports = "displayName": "yarn.resourcemanager.ha.enabled", "isReconfigurable": false, "recommendedValue": true, + "isOverridable": false, "value": true, "displayType": "checkbox", "category": "YARN", @@ -44,6 +46,7 @@ module.exports = "displayName": "yarn.resourcemanager.ha.rm-ids", "isReconfigurable": false, "recommendedValue": "rm1,rm2", + "isOverridable": false, "value": "rm1,rm2", "category": "YARN", "filename": "yarn-site", @@ -54,6 +57,7 @@ module.exports = "displayName": "yarn.resourcemanager.hostname.rm1", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "YARN", "filename": "yarn-site", @@ -65,6 +69,7 @@ module.exports = "displayName": "yarn.resourcemanager.webapp.address.rm1", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "YARN", "filename": "yarn-site", @@ -75,6 +80,7 @@ module.exports = "displayName": "yarn.resourcemanager.webapp.address.rm2", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "YARN", "filename": "yarn-site", @@ -85,6 +91,7 @@ module.exports = "displayName": "yarn.resourcemanager.webapp.https.address.rm1", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "YARN", "filename": "yarn-site", @@ -95,6 +102,7 @@ module.exports = "displayName": "yarn.resourcemanager.webapp.https.address.rm2", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "YARN", "filename": "yarn-site", @@ -106,6 +114,7 @@ module.exports = "displayName": "yarn.resourcemanager.hostname.rm2", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "YARN", "filename": "yarn-site", @@ -116,6 +125,7 @@ module.exports = "displayName": "yarn.resourcemanager.recovery.enabled", "isReconfigurable": false, "recommendedValue": true, + "isOverridable": false, "value": true, "displayType": "checkbox", "category": "YARN", @@ -127,6 +137,7 @@ module.exports = "displayName": "yarn.resourcemanager.store.class", "isReconfigurable": false, "recommendedValue": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore", + "isOverridable": false, "value": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore", "category": "YARN", "filename": "yarn-site", @@ -137,6 +148,7 @@ module.exports = "displayName": "yarn.resourcemanager.zk-address", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "YARN", "filename": "yarn-site", @@ -147,6 +159,7 @@ module.exports = "displayName": "yarn.resourcemanager.cluster-id", "isReconfigurable": false, "recommendedValue": "yarn-cluster", + "isOverridable": false, "value": "yarn-cluster", "category": "YARN", "filename": "yarn-site", @@ -157,6 +170,7 @@ module.exports = "displayName": "yarn.resourcemanager.ha.automatic-failover.zk-base-path", "isReconfigurable": false, "recommendedValue": "/yarn-leader-election", + "isOverridable": false, "value": "/yarn-leader-election", "category": "YARN", "filename": "yarn-site", @@ -169,6 +183,7 @@ module.exports = "description": "Comma separated yarn resourcemanager host addresses with port", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "HAWQ", "filename": "yarn-client", @@ -180,6 +195,7 @@ module.exports = "description": "Comma separated yarn resourcemanager scheduler addresses with port", "isReconfigurable": false, "recommendedValue": "", + "isOverridable": false, "value": "", "category": "HAWQ", "filename": "yarn-client", http://git-wip-us.apache.org/repos/asf/ambari/blob/6c71cd1f/ambari-web/app/messages.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js index 2203b17..17a9ef9 100644 --- a/ambari-web/app/messages.js +++ b/ambari-web/app/messages.js @@ -1326,7 +1326,8 @@ Em.I18n.translations = { 'admin.rm_highAvailability.wizard.step4.task1.title': 'Install Additional ResourceManager', 'admin.rm_highAvailability.wizard.step4.task2.title': 'Reconfigure YARN', 'admin.rm_highAvailability.wizard.step4.task3.title': 'Reconfigure HAWQ', - 'admin.rm_highAvailability.wizard.step4.task4.title': 'Start All Services', + 'admin.rm_highAvailability.wizard.step4.task4.title': 'Reconfigure HDFS', + 'admin.rm_highAvailability.wizard.step4.task5.title': 'Start All Services', 'admin.rm_highAvailability.wizard.step4.notice.inProgress':'Please wait while ResourceManager HA is being deployed.', 'admin.rm_highAvailability.wizard.step4.notice.completed':'ResourceManager HA has been enabled successfully.', http://git-wip-us.apache.org/repos/asf/ambari/blob/6c71cd1f/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js b/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js index 5339637..7002110 100644 --- a/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js +++ b/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js @@ -46,7 +46,6 @@ App.ConfigWithOverrideRecommendationParser = Em.Mixin.create(App.ConfigRecommend * @protected */ _updateOverride: function(config, recommendedValue, parentProperties, configGroup) { - debugger; var updateValue = this.allowUpdateProperty(parentProperties, Em.get(config, 'name'), Em.get(config, 'filename')); var override = config.getOverride(configGroup.get('name')); if (override) { http://git-wip-us.apache.org/repos/asf/ambari/blob/6c71cd1f/ambari-web/app/utils/configs/rm_ha_config_initializer.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/rm_ha_config_initializer.js b/ambari-web/app/utils/configs/rm_ha_config_initializer.js index 0255b27..9c36323 100644 --- a/ambari-web/app/utils/configs/rm_ha_config_initializer.js +++ b/ambari-web/app/utils/configs/rm_ha_config_initializer.js @@ -52,15 +52,31 @@ App.RmHaConfigInitializer = App.HaConfigInitializerClass.create(App.HostsBasedIn 'yarn.resourcemanager.webapp.https.address.rm1': this.getHostWithPortConfig('RESOURCEMANAGER', true, '', '', 'httpsWebAddressPort', true), 'yarn.resourcemanager.webapp.https.address.rm2': this.getHostWithPortConfig('RESOURCEMANAGER', false, '', '', 'httpsWebAddressPort', true), 'yarn.resourcemanager.ha': getRmHaHostsWithPort(8032), - 'yarn.resourcemanager.scheduler.ha': getRmHaHostsWithPort(8030) + 'yarn.resourcemanager.scheduler.ha': getRmHaHostsWithPort(8030), + 'hadoop.proxyuser.{{yarnUser}}.hosts': this.getComponentsHostsConfig(['RESOURCEMANAGER']) }; }.property(), initializerTypes: [ - {name: 'rm_hosts_with_port', method: '_initRmHaHostsWithPort'}, + {name: 'rm_hosts_with_port', method: '_initRmHaHostsWithPort'} ], /** + * @override + * @param {object} settings + */ + setup: function (settings) { + this._updateInitializers(settings); + }, + + /** + * @override + */ + cleanup: function () { + this._restoreInitializers(); + }, + + /** * Initializer for configs that should be updated with yarn resourcemanager ha host addresses with port * * @param {configProperty} configProperty http://git-wip-us.apache.org/repos/asf/ambari/blob/6c71cd1f/ambari-web/test/controllers/main/admin/highAvailability/resourceManager/step3_controller_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/controllers/main/admin/highAvailability/resourceManager/step3_controller_test.js b/ambari-web/test/controllers/main/admin/highAvailability/resourceManager/step3_controller_test.js index 5b34dd4..9a24cf3 100644 --- a/ambari-web/test/controllers/main/admin/highAvailability/resourceManager/step3_controller_test.js +++ b/ambari-web/test/controllers/main/admin/highAvailability/resourceManager/step3_controller_test.js @@ -71,6 +71,9 @@ describe('App.RMHighAvailabilityWizardStep3Controller', function () { }, 'yarn-site': { 'tag': 1 + }, + 'yarn-env': { + 'tag': 1 } } } @@ -78,7 +81,7 @@ describe('App.RMHighAvailabilityWizardStep3Controller', function () { 'serviceConfig': {} }); var data = App.ajax.send.args[0][0].data; - expect(data.urlParams).to.equal('(type=zoo.cfg&tag=1)|(type=yarn-site&tag=1)'); + expect(data.urlParams).to.equal('(type=zoo.cfg&tag=1)|(type=yarn-site&tag=1)|(type=yarn-env&tag=1)'); expect(data.serviceConfig).to.eql({}); }); @@ -221,6 +224,12 @@ describe('App.RMHighAvailabilityWizardStep3Controller', function () { } }, { + type: 'yarn-env', + properties: { + yarn_user: 'yarn' + } + }, + { type: 'yarn-site', properties: { 'yarn.resourcemanager.webapp.address': 'lclhst:1234', @@ -274,6 +283,9 @@ describe('App.RMHighAvailabilityWizardStep3Controller', function () { }), Em.Object.create({ name: 'yarn.resourcemanager.scheduler.ha' + }), + Em.Object.create({ + name: 'hadoop.proxyuser.yarn.hosts' }) ] }; @@ -347,10 +359,18 @@ describe('App.RMHighAvailabilityWizardStep3Controller', function () { it('yarn.resourcemanager.ha value', function () { expect(configs.configs.findProperty('name', 'yarn.resourcemanager.ha').get('value')).to.equal('h0:8032,h1:8032'); }); - it('yarn.resourcemanager.ha recommendedValud', function () { + it('yarn.resourcemanager.ha recommendedValue', function () { expect(configs.configs.findProperty('name', 'yarn.resourcemanager.scheduler.ha').get('recommendedValue')).to.equal('h0:8030,h1:8030'); }); + it('hadoop.proxyuser.yarn.hosts value', function () { + expect(configs.configs.findProperty('name', 'hadoop.proxyuser.yarn.hosts').get('value')).to.equal('h0,h1'); + }); + + it('hadoop.proxyuser.yarn.hosts recommendedValue', function () { + expect(configs.configs.findProperty('name', 'hadoop.proxyuser.yarn.hosts').get('recommendedValue')).to.equal('h0,h1'); + }); + }); });
