Repository: ambari Updated Branches: refs/heads/trunk 67c65aba9 -> 421770b27
AMBARI-11195. Need to be able to create customized graphs based on YARN queue-specific metrics. (jaimin) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/421770b2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/421770b2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/421770b2 Branch: refs/heads/trunk Commit: 421770b27048fedfdfbff7c9f659e5f1ade387b7 Parents: 67c65ab Author: Jaimin Jetly <[email protected]> Authored: Sun May 17 12:40:57 2015 -0700 Committer: Jaimin Jetly <[email protected]> Committed: Sun May 17 12:41:05 2015 -0700 ---------------------------------------------------------------------- ambari-web/app/assets/test/tests.js | 1 + .../service/widgets/create/step2_controller.js | 13 +- .../service/widgets/create/step3_controller.js | 10 +- .../service/widgets/create/wizard_controller.js | 67 +++++- .../widgets/create/wizard_controller_test.js | 235 +++++++++++++++++++ 5 files changed, 303 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/assets/test/tests.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/assets/test/tests.js b/ambari-web/app/assets/test/tests.js index 64456a9..95e9f35 100644 --- a/ambari-web/app/assets/test/tests.js +++ b/ambari-web/app/assets/test/tests.js @@ -87,6 +87,7 @@ var files = ['test/init_model_test', 'test/controllers/main/service/reassign/step4_controller_test', 'test/controllers/main/service/reassign/step6_controller_test', 'test/controllers/main/service/reassign/step7_controller_test', + 'test/controllers/main/service/widgets/create/wizard_controller_test', 'test/controllers/main/service/widgets/create/step1_controller_test', 'test/controllers/main/service/widgets/create/step2_controller_test', 'test/controllers/main/dashboard_test', http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js b/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js index 9c48872..65af9cd 100644 --- a/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js +++ b/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js @@ -343,14 +343,17 @@ App.WidgetWizardStep2Controller = Em.Controller.extend({ value = '${'; expression.data.forEach(function (element) { if (element.isMetric) { - metrics.push({ + var metricObj = { "name": element.name, "service_name": element.serviceName, "component_name": element.componentName, - "metric_path": element.metricPath, - "host_component_criteria": element.hostComponentCriteria, - "category": element.category - }); + "metric_path": element.metricPath + }; + if (element.hostComponentCriteria) { + metricObj.host_component_criteria = element.hostComponentCriteria; + } + metrics.push(metricObj); + } value += element.name; }, this); http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js b/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js index f946b74..24b32f8 100644 --- a/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js +++ b/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js @@ -122,15 +122,7 @@ App.WidgetWizardStep3Controller = Em.Controller.extend({ description: this.get('widgetDescription') || "", scope: this.get('widgetScope').toUpperCase(), author: this.get('widgetAuthor'), - "metrics": this.get('widgetMetrics').map(function (metric) { - return { - "name": metric.name, - "service_name": metric.serviceName, - "component_name": metric.componentName, - "metric_path": metric.metricPath, - "host_component_criteria": metric.hostComponentCriteria - } - }), + metrics: this.get('widgetMetrics'), values: this.get('widgetValues').map(function (value) { delete value.computedValue; return value; http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js b/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js index 08a5582..4c1dd06 100644 --- a/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js +++ b/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js @@ -180,17 +180,17 @@ App.WidgetWizardController = App.WizardController.extend({ * @returns {$.Deferred} */ loadAllMetrics: function () { - var widgetMetrics = this.getDBProperty('allMetrics'); + var allMetrics = this.getDBProperty('allMetrics'); var self = this; var dfd = $.Deferred(); - if (widgetMetrics.length === 0) { + if (allMetrics.length === 0) { this.loadAllMetricsFromServer(function () { dfd.resolve(self.get('content.allMetrics')); }); } else { - this.set('content.allMetrics', widgetMetrics); - dfd.resolve(widgetMetrics); + this.set('content.allMetrics', allMetrics); + dfd.resolve(allMetrics); } return dfd.promise(); }, @@ -223,23 +223,25 @@ App.WidgetWizardController = App.WizardController.extend({ var self = this; var result = []; var metrics = {}; - + var slaveComponents = App.StackServiceComponent.find().filterProperty('isSlave').mapProperty('componentName'); if (json) { json.items.forEach(function (service) { var data = service.artifacts[0].artifact_data[service.StackServices.service_name]; for (var componentName in data) { + var isSlave = slaveComponents.contains(componentName); for (var level in data[componentName]) { var metricTypes = data[componentName][level]; //Ganglia or JMX metricTypes.forEach(function (_metricType) { metrics = _metricType['metrics']['default']; var type = _metricType["type"].toUpperCase(); - if (!(type === 'JMX' && level.toUpperCase() === 'COMPONENT')) { + if (!((type === 'JMX' && level.toUpperCase() === 'COMPONENT') || (isSlave && level.toUpperCase() === 'HOSTCOMPONENT'))) { for (var widgetId in metrics) { + var _metrics = metrics[widgetId]; var metricObj = { widget_id: widgetId, - point_in_time: metrics[widgetId].pointInTime, - temporal: metrics[widgetId].temporal, - name: metrics[widgetId].name, + point_in_time: _metrics.pointInTime, + temporal: _metrics.temporal, + name: _metrics.name, level: level.toUpperCase(), type: type, component_name: componentName, @@ -256,9 +258,56 @@ App.WidgetWizardController = App.WizardController.extend({ } }, this); } + if (!!App.YARNService.find("YARN")) { + result = this.substitueQueueMetrics(result); + } this.save('allMetrics', result); }, + + /** + * @name substitueQueueMetrics + * @param metrics + * @return {Array} array of metric objects with regex substituted with actual metrics names + */ + substitueQueueMetrics: function (metrics) { + var result = []; + var queuePaths = App.YARNService.find("YARN").get("allQueueNames"); + var queueNames = []; + var queueMetricName; + queuePaths.forEach(function (_queuePath) { + var queueName = _queuePath.replace(/\//g, "."); + queueNames.push(queueName); + }, this); + var regexpForAMS = new RegExp("^yarn.QueueMetrics.Queue=\\(\\.\\+\\).*$"); + var regexpForJMX = new RegExp("\\(\\.\\+\\)", "g"); + var replaceRegexForMetricName = regexpForJMX; + var replaceRegexForMetricPath = new RegExp("\\$\\d\\..*\\)(?=\\/)", "g"); + metrics.forEach(function (_metric) { + var isAMSQueueMetric = regexpForAMS.test(_metric.name); + var isJMXQueueMetrics = regexpForJMX.test(_metric.name); + if ((_metric.type === 'GANGLIA' && isAMSQueueMetric) || (_metric.type === 'JMX' && isJMXQueueMetrics)) { + queuePaths.forEach(function (_queuePath) { + queueMetricName = ''; + if (_metric.type === 'GANGLIA') { + queueMetricName = _queuePath.replace(/\//g, "."); + } else if (_metric.type === 'JMX') { + _queuePath.split("/").forEach(function(_metricName, index){ + queueMetricName = queueMetricName + ',q' + index + '=' + _metricName; + }, this); + } + var metricName = _metric.name.replace(replaceRegexForMetricName, queueMetricName); + var newMetricPath = _metric.widget_id.replace(replaceRegexForMetricPath, _queuePath); + var newQueueMetric = $.extend(true, {}, _metric, {name: metricName, widget_id: newMetricPath}); + result.pushObject(newQueueMetric); + }, this); + } else { + result.pushObject(_metric); + } + }, this); + return result; + }, + /** * * @param metricObj {Object} http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js b/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js new file mode 100644 index 0000000..83ce39f --- /dev/null +++ b/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +App = require('app'); + +require('controllers/main/service/widgets/create/wizard_controller'); + + +describe('App.WidgetWizardController', function () { + var controller; + + /** + * tests the function with following hierarchical queue scenario + * root + * | + * queue1 + * / \ + * queue2 queue3 + * + */ + describe("#substitueQueueMetrics", function () { + beforeEach(function () { + controller = App.WidgetWizardController.create(); + sinon.stub(App.YARNService, 'find', function (k) { + if ('YARN' === k) return Em.Object.create({ + 'allQueueNames': ["root", "root/queue1", "root/queue1/queue2", "root/queue1/queue3"] + }); + }); + }); + afterEach(function () { + controller = ''; + App.YARNService.find.restore(); + }); + + + var testCases = [ + { + msg: 'AMS Queue metric with regex as name and regex as path should be replaced with actual metric name and path of all existing queues', + inputMetrics: [ + { + component_name: 'RESOURCEMANAGER', + level: 'COMPONENT', + name: 'yarn.QueueMetrics.Queue=(.+).AppsFailed', + point_in_time: false, + service_name: 'YARN', + temporal: true, + type: 'GANGLIA', + widget_id: 'metrics/yarn/Queue/$1.replaceAll("([.])","/")/AppsFailed' + } + ], + expectedResult: [ + { + "component_name": "RESOURCEMANAGER", + "level": "COMPONENT", + "name": "yarn.QueueMetrics.Queue=root.AppsFailed", + "point_in_time": false, + "service_name": "YARN", + "temporal": true, + "type": "GANGLIA", + "widget_id": "metrics/yarn/Queue/root/AppsFailed" + }, + { + "component_name": "RESOURCEMANAGER", + "level": "COMPONENT", + "name": "yarn.QueueMetrics.Queue=root.queue1.AppsFailed", + "point_in_time": false, + "service_name": "YARN", + "temporal": true, + "type": "GANGLIA", + "widget_id": "metrics/yarn/Queue/root/queue1/AppsFailed" + }, + { + "component_name": "RESOURCEMANAGER", + "level": "COMPONENT", + "name": "yarn.QueueMetrics.Queue=root.queue1.queue2.AppsFailed", + "point_in_time": false, + "service_name": "YARN", + "temporal": true, + "type": "GANGLIA", + "widget_id": "metrics/yarn/Queue/root/queue1/queue2/AppsFailed" + }, + { + "component_name": "RESOURCEMANAGER", + "level": "COMPONENT", + "name": "yarn.QueueMetrics.Queue=root.queue1.queue3.AppsFailed", + "point_in_time": false, + "service_name": "YARN", + "temporal": true, + "type": "GANGLIA", + "widget_id": "metrics/yarn/Queue/root/queue1/queue3/AppsFailed" + } + ] + }, + { + msg: 'JMX Queue metric with regex as name and regex as path should be replaced with actual metric name and path of all existing queues', + inputMetrics: [ + { + component_name: 'RESOURCEMANAGER', + host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE', + level: 'HOSTCOMPONENT', + name: 'Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsFailed', + point_in_time: true, + service_name: 'YARN', + temporal: false, + type: 'JMX', + widget_id: 'metrics/yarn/Queue/$1.replaceAll(",q(\d+)=","/").substring(1)/AppsFailed' + } + ], + expectedResult: [ + { + component_name: 'RESOURCEMANAGER', + host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE', + level: 'HOSTCOMPONENT', + name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root.AppsFailed', + point_in_time: true, + service_name: 'YARN', + temporal: false, + type: 'JMX', + widget_id: 'metrics/yarn/Queue/root/AppsFailed' + }, + { + component_name: 'RESOURCEMANAGER', + host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE', + level: 'HOSTCOMPONENT', + name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1.AppsFailed', + point_in_time: true, + service_name: 'YARN', + temporal: false, + type: 'JMX', + widget_id: 'metrics/yarn/Queue/root/queue1/AppsFailed' + }, + { + component_name: 'RESOURCEMANAGER', + host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE', + level: 'HOSTCOMPONENT', + name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1,q2=queue2.AppsFailed', + point_in_time: true, + service_name: 'YARN', + temporal: false, + type: 'JMX', + widget_id: 'metrics/yarn/Queue/root/queue1/queue2/AppsFailed' + }, + { + component_name: 'RESOURCEMANAGER', + host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE', + level: 'HOSTCOMPONENT', + name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1,q2=queue3.AppsFailed', + point_in_time: true, + service_name: 'YARN', + temporal: false, + type: 'JMX', + widget_id: 'metrics/yarn/Queue/root/queue1/queue3/AppsFailed' + } + ] + }, + { + msg: 'AMS Queue metric without regex in name and path should retain same name and path', + inputMetrics: [ + { + component_name: 'RESOURCEMANAGER', + level: 'COMPONENT', + name: 'yarn.QueueMetrics.Queue.Clustermetrics.AppsFailed', + point_in_time: false, + service_name: 'YARN', + temporal: true, + type: 'GANGLIA', + widget_id: 'metrics/yarn/Queue/Clustermetrics/AppsFailed' + } + ], + expectedResult: [ + { + component_name: 'RESOURCEMANAGER', + level: 'COMPONENT', + name: 'yarn.QueueMetrics.Queue.Clustermetrics.AppsFailed', + point_in_time: false, + service_name: 'YARN', + temporal: true, + type: 'GANGLIA', + widget_id: 'metrics/yarn/Queue/Clustermetrics/AppsFailed' + } + ] + }, + { + msg: 'JMX Queue metric without regex in name and path should retain same name and path', + inputMetrics: [ + { + component_name: 'RESOURCEMANAGER', + host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE', + level: 'HOSTCOMPONENT', + name: 'Hadoop:service=ResourceManager,name=QueueMetrics.clusterMetric.AppsFailed', + point_in_time: true, + service_name: 'YARN', + temporal: false, + type: 'JMX', + widget_id: 'metrics/yarn/Queue/clusterMetric/AppsFailed' + } + ], + expectedResult: [ + { + component_name: 'RESOURCEMANAGER', + host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE', + level: 'HOSTCOMPONENT', + name: 'Hadoop:service=ResourceManager,name=QueueMetrics.clusterMetric.AppsFailed', + point_in_time: true, + service_name: 'YARN', + temporal: false, + type: 'JMX', + widget_id: 'metrics/yarn/Queue/clusterMetric/AppsFailed' + } + ] + } + ]; + testCases.forEach(function (_testCase) { + it(_testCase.msg, function () { + var result = controller.substitueQueueMetrics(_testCase.inputMetrics); + expect(JSON.stringify(result)).to.equal(JSON.stringify(_testCase.expectedResult)); + }); + }); + }); +});
