TEZ-2817. Tez UI: update in progress counter data for the dag vertices and tasks table (sree via pramachandran)
(cherry picked from commit b3ad59578ba801a44c8e97b39534cf5efb90c1f1) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6b05e196 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6b05e196 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6b05e196 Branch: refs/heads/branch-0.7 Commit: 6b05e19653f965d336fdfde5f829357c728c6791 Parents: 64526ea Author: Prakash Ramachandran <[email protected]> Authored: Mon Sep 14 23:00:13 2015 +0530 Committer: Prakash Ramachandran <[email protected]> Committed: Mon Sep 14 23:18:49 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + tez-ui/src/main/webapp/app/scripts/app.js | 27 ++++--- .../app/scripts/controllers/dag_controller.js | 14 +++- .../webapp/app/scripts/controllers/dag_tasks.js | 29 +++----- .../app/scripts/controllers/dag_vertices.js | 23 +++++- .../scripts/helpers/entity-array-pollster.js | 26 ++++++- .../src/main/webapp/app/scripts/helpers/misc.js | 75 +++++++++++++++++++- .../src/main/webapp/app/scripts/models/dag.js | 7 +- 8 files changed, 165 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/6b05e196/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index e7eb8d1..184552f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ Release 0.7.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2817. Tez UI: update in progress counter data for the dag vertices and tasks table TEZ-2813. Tez UI: add counter data for rest api calls to AM Web Services v2 TEZ-2660. Tez UI: need to show application page even if system metrics publish is disabled. TEZ-2787. Tez AM should have java.io.tmpdir=./tmp to be consistent with tasks http://git-wip-us.apache.org/repos/asf/tez/blob/6b05e196/tez-ui/src/main/webapp/app/scripts/app.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/app.js b/tez-ui/src/main/webapp/app/scripts/app.js index 13c08cb..2ad5fdd 100644 --- a/tez-ui/src/main/webapp/app/scripts/app.js +++ b/tez-ui/src/main/webapp/app/scripts/app.js @@ -206,16 +206,20 @@ App.ready = function () { App.VertexInfoAdapter = App.AMInfoAdapter.extend({ namespace: App.Configs.restNamespace.aminfoV2, - ajax: function(url, method, hash) { - var options = hash.data || {}; - url = url.replace('__app_id__', options.appId) - .fmt(options.dagIdx); - delete options['dagIdx']; - delete options['appId']; - return this._super(url, method, hash); + findQuery: function(store, type, query) { + var record = query.metadata; + delete query.metadata; + return this.ajax( + this.buildURL(Ember.String.pluralize(type.typeKey), + record.vertexID, Em.Object.create(record)), 'GET', { data: query}); }, - pathForType: function() { - return 'verticesInfo?dagID=%@'; + buildURL: function(type, id, record) { + var url = this._super(type, undefined, record); + return url.replace('__app_id__', record.get('appID')) + .fmt(record.get('dagID'), id, record.get('counters')); + }, + pathForType: function(typeName) { + return 'verticesInfo?dagID=%@&vertexID=%@&counters=%@'; } }); @@ -231,12 +235,13 @@ App.ready = function () { buildURL: function(type, id, record) { var url = this._super(type, undefined, record); return url.replace('__app_id__', record.get('appID')) - .fmt(record.get('dagID'), id); + .fmt(record.get('dagID'), id, record.get('counters')); }, pathForType: function(typeName) { - return 'tasksInfo?dagID=%@&taskID=%@'; + return 'tasksInfo?dagID=%@&taskID=%@&counters=%@'; } }); + }; $.ajaxPrefilter(function(options, originalOptions, jqXHR) { http://git-wip-us.apache.org/repos/asf/tez/blob/6b05e196/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js index 8804813..cd750d0 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js @@ -80,6 +80,7 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, { ["loading", "id", "model.status"].forEach(function(item) { Em.addObserver(that, item, that.startAMInfoUpdateService); }); + that.startAMInfoUpdateService(); } } }); @@ -121,9 +122,12 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, { if (!dagId) return; - verticesInfoLoader = this.store.find('vertexInfo', { - appId: that.get('applicationId'), - dagIdx: that.get('idx') + verticesInfoLoader = this.store.findQuery('vertexInfo', { + metadata: { + appID: that.get('applicationId'), + dagID: that.get('idx'), + counters: App.get('vertexCounters') + } }); verticesInfoLoader.then(function(verticesInfo) { @@ -156,6 +160,10 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, { Em.addObserver(that, item, that.stopAMInfoUpdateService); }); } + else { + that.updateAMDagInfo(); + that.updateAMVerticesInfo(); + } }, dostopAMInfoUpdateService: function() { http://git-wip-us.apache.org/repos/asf/tez/blob/6b05e196/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js index 0bd9df0..567b608 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js @@ -39,8 +39,8 @@ App.DagTasksController = App.TablePageController.extend({ }, pollsterControl: function () { - if(this.get('controllers.dag.status') == 'RUNNING' && - this.get('controllers.dag.amWebServiceVersion') != '1' && + if(this.get('status') == 'RUNNING' && + this.get('amWebServiceVersion') != '1' && !this.get('loading') && this.get('isActive') && this. get('rowsDisplayed.length') > 0) { @@ -49,34 +49,27 @@ App.DagTasksController = App.TablePageController.extend({ else { this.get('pollster').stop(); } - }.observes('controllers.dag.status', - 'controllers.dag.amWebServiceVersion', - 'rowsDisplayed', - 'loading', - 'isActive'), - - reset: function () { - this.set('pollster.options', null); - }, + }.observes('status', 'amWebServiceVersion', 'rowsDisplayed', 'loading', 'isActive'), pollsterOptionsObserver: function () { var rows = this.get('rowsDisplayed'); this.set('pollster.targetRecords', rows); - rows = rows.filter(function (row) { - return row.get('status') != 'SUCCEEDED'; - }); - this.set('pollster.options', (rows && rows.length) ? { - appID: this.get('controllers.dag.model.applicationId'), - dagID: this.get('controllers.dag.model.idx'), + appID: this.get('applicationId'), + dagID: this.get('idx'), + counters: this.get('countersDisplayed'), taskID: rows.map(function (row) { var taskIndex = App.Helpers.misc.getIndexFromId(row.get('id')), vertexIndex = App.Helpers.misc.getIndexFromId(row.get('vertexID')); return '%@_%@'.fmt(vertexIndex, taskIndex); }).join(',') } : null); - }.observes('controllers.dag.model.applicationId', 'controllers.dag.model.idx', 'rowsDisplayed'), + }.observes('applicationId', 'idx', 'rowsDisplayed', 'counters'), + + countersDisplayed: function () { + return App.Helpers.misc.getCounterQueryParam(this.get('columns')); + }.property('columns'), beforeLoad: function () { var dagController = this.get('controllers.dag'), http://git-wip-us.apache.org/repos/asf/tez/blob/6b05e196/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js index ea25818..5f60d5d 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js @@ -26,6 +26,11 @@ App.DagVerticesController = App.TablePageController.extend({ cacheDomain: Ember.computed.alias('controllers.dag.id'), + pollster: {}, + amInfoUpdateServiceObserver: function () { + this.set('pollster.isRunning', !!this.get('amInfoUpdateService')); + }.observes('amInfoUpdateService').on('init'), + beforeLoad: function () { var dagController = this.get('controllers.dag'), model = dagController.get('model'); @@ -69,7 +74,8 @@ App.DagVerticesController = App.TablePageController.extend({ metadata: { appId: that.get('applicationId'), dagIdx: that.get('idx'), - vertexIds: runningVerticesIdx.join(',') + vertexIds: runningVerticesIdx.join(','), + counters: counters } }).then(function(vertexProgressInfo) { that.set('controllers.dag.amVertexInfo', vertexProgressInfo); @@ -79,10 +85,18 @@ App.DagVerticesController = App.TablePageController.extend({ } }, + _onColumnChange: function () { + App.set('vertexCounters', App.Helpers.misc.getCounterQueryParam(this.get('columns'))); + }.observes('columns').on('init'), + overlayVertexInfo: function(vertex, amVertexInfo) { if (Em.isNone(amVertexInfo) || Em.isNone(vertex)) return; amVertexInfo.set('_amInfoLastUpdatedTime', moment()); vertex.setProperties(amVertexInfo.getProperties('status', 'progress', '_amInfoLastUpdatedTime')); + + vertex.set('counterGroups', + App.Helpers.misc.mergeCounterInfo(vertex.get('counterGroups'), amVertexInfo.get('counters')).slice(0) + ); }, updateVertexInfo: function() { @@ -152,6 +166,13 @@ App.DagVerticesController = App.TablePageController.extend({ } }, { + id: 'progress', + headerCellName: 'Progress', + contentPath: 'progress', + observePath: true, + templateName: 'components/basic-table/progress-cell' + }, + { id: 'startTime', headerCellName: 'Start Time', contentPath: 'startTime', http://git-wip-us.apache.org/repos/asf/tez/blob/6b05e196/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js b/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js index 040ec8f..fb3dd76 100644 --- a/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js +++ b/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js @@ -64,6 +64,10 @@ App.Helpers.EntityArrayPollster = App.Helpers.Pollster.extend({ } }, + _optionObserver: function () { + Em.run.later(this, this.onPoll, 10); + }.observes('options'), + _callIfRunning: function (that, funName) { return function (data) { var fun = that.get(funName); @@ -93,9 +97,27 @@ App.Helpers.EntityArrayPollster = App.Helpers.Pollster.extend({ if(polledRecords && targetRecords) { targetRecords.forEach(function (row) { - var info = polledRecords.findBy('id', row.get('id')); - if(info) { + var info = polledRecords.findBy('id', row.get('id')), + merge = !!info; + + if(merge && row.get('progress') && info.get('progress')) { + if(row.get('progress') >= info.get('progress')) { + merge = false; + } + } + + if(merge) { row.setProperties(info.getProperties.apply(info, mergeProperties)); + + if(info.get('counters')) { + row.set('counterGroups', + App.Helpers.misc.mergeCounterInfo( + row.get('counterGroups'), + info.get('counters') + ).slice(0) + ); + } + } }); } http://git-wip-us.apache.org/repos/asf/tez/blob/6b05e196/tez-ui/src/main/webapp/app/scripts/helpers/misc.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js index 223b686..ddd7d6a 100644 --- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js +++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js @@ -47,6 +47,8 @@ App.Helpers.misc = { return 'success'; case 'UNDEFINED': return 'unknown'; + case 'SCHEDULED': + return 'schedule'; default: return 'submitted'; } @@ -158,7 +160,7 @@ App.Helpers.misc = { * @param counterConfigs Array * @return Normalized configurations */ - normalizeCounterConfigs: function (counterConfigs) { + normalizeCounterConfigs: function (counterConfigs, inProgress) { return counterConfigs.map(function (configuration) { var groupName = configuration.counterGroupName || configuration.groupId, counterName = configuration.counterName || configuration.counterId; @@ -169,6 +171,12 @@ App.Helpers.misc = { ); configuration.id = '%@/%@'.fmt(groupName, counterName), + configuration.observePath = true; + configuration.contentPath = 'counterGroups'; + configuration.counterGroupName = groupName; + configuration.counterName = counterName; + configuration.searchAndSortable = !inProgress; + configuration.getSortValue = App.Helpers.misc.getCounterCellContent; configuration.getCellContent = configuration.getSearchValue = App.Helpers.misc.getCounterCellContentFormatted; @@ -176,6 +184,71 @@ App.Helpers.misc = { }); }, + getCounterQueryParam: function (columns) { + var counterHash = {}, + counters = []; + + columns.forEach(function (column) { + var groupName = column.get('counterGroupName'), + counterName = column.get('counterName'); + if(column.get('contentPath') == 'counterGroups') { + counterHash[groupName] = counterHash[groupName] || []; + counterHash[groupName].push(counterName); + } + }); + for(var groupName in counterHash) { + counters.push('%@/%@'.fmt(groupName, counterHash[groupName].join(','))); + } + + return counters.join(';'); + }, + + /* + * Merges counter information from AM counter object into ATS counters array + */ + mergeCounterInfo: function (targetATSCounters, sourceAMCounters) { + var atsCounters, atsCounter, + counters; + + targetATSCounters = targetATSCounters || []; + + try{ + for(var counterGroupName in sourceAMCounters) { + counters = sourceAMCounters[counterGroupName], + atsCounters = targetATSCounters.findBy('counterGroupName', counterGroupName); + if(!atsCounters) { + atsCounters = []; + targetATSCounters.push({ + counterGroupName: counterGroupName, + counterGroupDisplayName: counterGroupName, + counters: atsCounters + }); + } + else { + atsCounters = atsCounters.counters; + } + for(var counterName in counters) { + atsCounter = atsCounters.findBy('counterName', counterName); + if(atsCounter) { + Em.set(atsCounter, 'counterValue', counters[counterName]); + } + else { + atsCounters.push({ + "counterName": counterName, + "counterDisplayName": counterName, + "counterValue": counters[counterName] + }); + } + } + } + } + catch(e){ + Em.Logger.info("Counter merge failed", e); + } + + return targetATSCounters; + }, + /* * Creates column definitions form configuration object array * @param columnConfigs Array http://git-wip-us.apache.org/repos/asf/tez/blob/6b05e196/tez-ui/src/main/webapp/app/scripts/models/dag.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/models/dag.js b/tez-ui/src/main/webapp/app/scripts/models/dag.js index c00e591..3e48cfe 100644 --- a/tez-ui/src/main/webapp/app/scripts/models/dag.js +++ b/tez-ui/src/main/webapp/app/scripts/models/dag.js @@ -138,6 +138,8 @@ App.Vertex = App.AbstractEntity.extend({ return !!f && f > 0; }.property('failedTasks'), + progress: DS.attr('number'), + /** * Vertex type has to be one of the types defined in 'App.VertexType' * @return {string} @@ -428,12 +430,15 @@ App.VertexInfo = DS.Model.extend({ killedTaskAttempts: DS.attr('number'), pendingTasks: function() { return this.get('totalTasks') - this.get('runningTasks') - this.get('succeededTasks'); - }.property('totalTasks', 'runningTasks', 'succeededTasks') + }.property('totalTasks', 'runningTasks', 'succeededTasks'), + + counters: DS.attr('object') }); App.TaskInfo = DS.Model.extend({ progress: DS.attr('number'), status: DS.attr('string'), + counters: DS.attr('object') }); App.KVDatum = DS.Model.extend({
