Repository: tez Updated Branches: refs/heads/master 69e3de818 -> 5af06047f
TEZ-2346. TEZ-UI: Lazy load other info / counter data (Sreenath Somarajapuram via jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5af06047 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5af06047 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5af06047 Branch: refs/heads/master Commit: 5af06047f03eafbfa97ba0b5213b79bb6e38165e Parents: 69e3de8 Author: Jonathan Eagles <[email protected]> Authored: Mon Dec 7 17:26:58 2015 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Mon Dec 7 17:26:58 2015 -0600 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../app/scripts/controllers/dags_controller.js | 136 ++++++++++++------- .../src/main/webapp/app/scripts/helpers/misc.js | 5 + .../app/scripts/mixins/paginated_content.js | 14 +- .../app/scripts/models/TimelineRestAdapter.js | 66 ++++++++- .../src/main/webapp/app/scripts/models/dag.js | 2 + 6 files changed, 168 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index b8619dc..9031566 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES TEZ-2949. Allow duplicate dag names within session for Tez. ALL CHANGES: + TEZ-2346. TEZ-UI: Lazy load other info / counter data TEZ-2975. Bump up apache commons dependency. TEZ-2970. Re-localization in TezChild does not use correct UGI. TEZ-2968. Counter limits exception causes AM to crash. @@ -273,6 +274,7 @@ INCOMPATIBLE CHANGES TEZ-2949. Allow duplicate dag names within session for Tez. ALL CHANGES + TEZ-2346. TEZ-UI: Lazy load other info / counter data TEZ-2975. Bump up apache commons dependency. TEZ-2970. Re-localization in TezChild does not use correct UGI. TEZ-2968. Counter limits exception causes AM to crash. http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js index ad9d636..7a52837 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js @@ -25,6 +25,8 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C pageSubTitle: 'All Tez DAGs', + columnSelectorMessage: 'Logs and counter columns needs more time to load.', + // query parameters supported through url. The same named variables in this controller get // bound automatically to the ones defined in the route. queryParams: { @@ -35,6 +37,8 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C dagName_filter: 'dag_name' }, + _loadedAllData: false, + fromID: null, status_filter: null, @@ -63,6 +67,25 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C })); }.observes('status_filter', 'user_filter', 'appId_filter', 'dagName_filter', 'id_filter'), + _otherInfoFieldsVisible: function () { + var visibleColumns = this.get('visibleColumnIds') || {}, + columnIds; + + if(visibleColumns['logs']) { + return true; + } + + columnIds = Object.keys(visibleColumns); + for(var i = 0, length = columnIds.length, id; i < length; i++) { + id = columnIds[i]; + if(visibleColumns[id] && id.indexOf('/') != -1) { + return true; + } + } + + return false; + }.property('visibleColumnIds'), + _filterVisiblilityObserver: function () { var visibleFilters = Em.Object.create(); this.get('columns').forEach(function (column) { @@ -103,25 +126,50 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C store.unloadAll(childEntityType); store.unloadAll('dagProgress'); - if(this.id_filter) { - finder = store.find(childEntityType, this.id_filter).then(function (entity) { - return ( + that.set('_loadedAllData', false); + function loadAllData() { + return store.findQuery(childEntityType, that.getFilterProperties()).then(function (entities) { + that.set('_loadedAllData', true); + return entities; + }); + } + + function setEntities(entities) { + that.set('entities', entities); + that.set('loading', false); + + return entities; + } + + if(that.id_filter) { + finder = store.find(childEntityType, that.id_filter).then(function (entity) { + var entities = ( (that.dagName_filter && entity.get('name') != that.dagName_filter) || (that.appId_filter && entity.get('applicationId') != that.appId_filter) || (that.user_filter && entity.get('user') != that.user_filter) || (that.status_filter && entity.get('status') != that.status_filter) ) ? [] : [entity]; - }).catch(function () { - return []; + + return setEntities(entities); }); } else { - finder = store.findQuery(childEntityType, this.getFilterProperties()); + // Query just basic data + finder = store.findQuery(childEntityType, that.getFilterProperties('events,primaryfilters')); + finder = finder.then(setEntities).then(function (entities) { + + // If countersVisible lazy load counters + return that.get('_otherInfoFieldsVisible') ? new Promise(function (fullfill) { + setTimeout(fullfill, 100); // Lazyload delay + }).then(loadAllData) : entities; + + }).catch(function () { + // Basic data query failed, probably YARN-3530 fix is not in ATS. Load all data. + return loadAllData().then(setEntities); + }); } - finder.then(function(entities){ - that.set('entities', entities); - that.set('loading', false); + finder = finder.then(function(entities){ entities.forEach(function (dag) { var appId = dag.get('applicationId'); @@ -133,8 +181,7 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C app.get('status'), app.get('finalStatus') )); - }).catch(function(error) {}) - .finally(function () { + }).finally(function (entities) { if(dag.get('status') === 'RUNNING') { App.Helpers.misc.removeRecord(store, 'dagProgress', dag.get('id')); store.find('dagProgress', dag.get('id'), { @@ -155,13 +202,20 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C }); } }); + }).catch(function(error){ Em.Logger.error(error); var err = App.Helpers.misc.formatError(error, defaultErrMsg); var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg); App.Helpers.ErrorBar.getInstance().show(msg, err.details); }); - }.observes('fields'), + }, + + _onCountersVisible: function () { + if(this.get('_otherInfoFieldsVisible') && !this.get('_loadedAllData')) { + Em.run.once(this, this.loadEntities); + } + }.observes('_otherInfoFieldsVisible'), actions : { filterUpdated: function() { @@ -185,24 +239,6 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C defaultColumnConfigs: function () { var store = this.get('store'); - function onProgressChange() { - var progress = this.get('dag.progress'), - pct; - if (Ember.typeOf(progress) === 'number') { - pct = App.Helpers.number.fractionToPercentage(progress); - this.set('progress', pct); - } - } - - function onStatusChange() { - var status = this.get('dag.status'); - this.setProperties({ - status: status, - statusIcon: App.Helpers.misc.getStatusClassForEntity(status, - this.get('dag.hasFailedTaskAttempts')) - }); - } - return [ { id: 'dagName', @@ -234,24 +270,26 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C headerCellName: 'Status', templateName: 'components/basic-table/status-cell', enableFilter: true, + contentPath: 'status', + observePath: true, getCellContent: function(row) { - var status = row.get('status'), - content = Ember.Object.create({ - dag: row, - status: status, - statusIcon: App.Helpers.misc.getStatusClassForEntity(status, - row.get('hasFailedTaskAttempts')) - }); - - if(status == 'RUNNING') { - row.addObserver('progress', content, onProgressChange); - row.addObserver('status', content, onStatusChange); - } - - return content; + var status = row.get('status'); + return { + status: status, + statusIcon: App.Helpers.misc.getStatusClassForEntity(status, + row.get('hasFailedTaskAttempts')) + }; } }, { + id: 'progress', + headerCellName: 'Progress', + contentPath: 'progress', + enableFilter: true, + observePath: true, + templateName: 'components/basic-table/progress-cell' + }, + { id: 'startTime', headerCellName: 'Start Time', contentPath: 'startTime', @@ -303,11 +341,15 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C id: 'logs', headerCellName: 'Logs', templateName: 'components/basic-table/multi-logs-cell', + contentPath: 'containerLogs', + observePath: true, getCellContent: function(row) { - var content = { - logs: row.get('containerLogs') + var containerLogs = row.get('containerLogs'); + return containerLogs ? { + logs: containerLogs + } : { + isPending: true }; - return content; } } ]; http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/helpers/misc.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js index b6fd690..8e19686 100644 --- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js +++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js @@ -302,6 +302,11 @@ App.Helpers.misc = { * @return value */ getCounterCellContentFormatted: function (row) { + if(row.get(this.get('contentPath')) == undefined) { + return { + isPending: true + }; + } var value = App.Helpers.misc.getCounterCellContent.call(this, row); return App.Helpers.number.formatNumThousands(value); }, http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js index 3acce39..d393df7 100644 --- a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js +++ b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js @@ -44,11 +44,13 @@ App.PaginatedContentMixin = Em.Mixin.create({ entities: [], _paginationFilters: {}, - loading: true, + loading: false, load: function() { - this.resetNavigation(); - this.loadEntities(); + if(!this.get('loading')) { + this.resetNavigation(); + this.loadEntities(); + } }.observes('rowCount'), lastPage: function () { @@ -149,7 +151,7 @@ App.PaginatedContentMixin = Em.Mixin.create({ return p.join(','); }, - getFilterProperties: function() { + getFilterProperties: function(fields) { var params = { limit: Math.min(this.rowCount, this.get('maxRowCount')) + 1 }; @@ -191,6 +193,10 @@ App.PaginatedContentMixin = Em.Mixin.create({ params['fromId'] = this.get('fromID'); } + if (fields) { + params['fields'] = fields; + } + return params; }, }); http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js b/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js index b4a2062..71f96df 100644 --- a/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js +++ b/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js @@ -85,27 +85,74 @@ App.TimelineSerializer = DS.RESTSerializer.extend({ } }); +function getStatus(source) { + var status = Em.get(source, 'otherinfo.status') || Em.get(source, 'primaryfilters.status.0'), + event = source.events; + + if(!status && event) { + if(event.findBy('eventtype', 'DAG_STARTED')) { + status = 'RUNNING'; + } + } + + return status; +} var timelineJsonToDagMap = { id: 'entity', submittedTime: 'starttime', - startTime: 'otherinfo.startTime', - endTime: 'otherinfo.endTime', + + startTime: { + custom: function(source) { + var time = Em.get(source, 'otherinfo.startTime'), + event = source.events; + + if(!time && event) { + event = event.findBy('eventtype', 'DAG_STARTED'); + if(event) { + time = event.timestamp; + } + } + + return time; + } + }, + endTime: { + custom: function(source) { + var time = Em.get(source, 'otherinfo.endTime'), + event = source.events; + + if(!time && event) { + event = event.findBy('eventtype', 'DAG_FINISHED'); + if(event) { + time = event.timestamp; + } + } + + return time; + } + }, + name: 'primaryfilters.dagName.0', user: 'primaryfilters.user.0', - status: 'otherinfo.status', + status: { + custom: getStatus + }, progress: { custom: function(source) { - return Em.get(source, 'otherinfo.status') == 'SUCCEEDED' ? 1 : null; + var status = getStatus(source); + return status == 'SUCCEEDED' ? 1 : null; } }, containerLogs: { custom: function(source) { - var containerLogs = []; var otherinfo = Em.get(source, 'otherinfo'); + if(!otherinfo) { + return undefined; + } for (var key in otherinfo) { if (key.indexOf('inProgressLogsURL_') === 0) { var logs = Em.get(source, 'otherinfo.' + key); @@ -130,7 +177,14 @@ var timelineJsonToDagMap = { numFailedTasks: 'otherinfo.numFailedTasks', diagnostics: 'otherinfo.diagnostics', - counterGroups: 'otherinfo.counters.counterGroups', + counterGroups: { + custom: function(source) { + var otherinfo = source.otherinfo; + if(otherinfo) { + return Em.get(otherinfo, 'counters.counterGroups') || []; + } + } + }, planName: 'otherinfo.dagPlan.dagName', planVersion: 'otherinfo.dagPlan.version', http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/models/dag.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/models/dag.js b/tez-ui/src/main/webapp/app/scripts/models/dag.js index ff4a388..3ea76d2 100644 --- a/tez-ui/src/main/webapp/app/scripts/models/dag.js +++ b/tez-ui/src/main/webapp/app/scripts/models/dag.js @@ -21,6 +21,8 @@ App.Dag = App.AbstractEntity.extend({ return App.Helpers.misc.getDagIndexFromDagId(this.get('id')); }.property('id'), + progress: DS.attr('number'), + submittedTime: DS.attr('number'), // start time of the entity
