Repository: tez Updated Branches: refs/heads/master 77dfdc0dd -> 98706f271
TEZ-2933. Tez UI: Load application details from RM when available (sree) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/98706f27 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/98706f27 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/98706f27 Branch: refs/heads/master Commit: 98706f271f2a0e84866b82e5d8555c970f151872 Parents: 77dfdc0 Author: Sreenath Somarajapuram <s...@apache.org> Authored: Wed Nov 18 19:04:21 2015 +0530 Committer: Sreenath Somarajapuram <s...@apache.org> Committed: Wed Nov 18 19:04:21 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../app/scripts/controllers/dag_controller.js | 11 ++-- .../app/scripts/controllers/dags_controller.js | 64 +++++++++----------- .../task_task_attempts_controller.js | 9 ++- .../scripts/controllers/tez-app-controller.js | 7 +-- .../controllers/tez-app-index-controller.js | 7 +-- .../scripts/controllers/vertex_controller.js | 13 ++-- .../src/main/webapp/app/scripts/helpers/misc.js | 20 ++++++ .../app/scripts/models/TimelineRestAdapter.js | 17 +++++- .../src/main/webapp/app/scripts/models/dag.js | 20 ++++-- tez-ui/src/main/webapp/app/scripts/router.js | 2 +- .../main/webapp/app/templates/tez-app/index.hbs | 6 +- 12 files changed, 103 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 6be2b99..4163fa9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-2679. Admin forms of launch env settings ALL CHANGES: + TEZ-2933. Tez UI: Load application details from RM when available TEZ-2908. Tez UI: Errors are logged, but not displayed in the UI when AM fetch fails TEZ-2923. Tez Live UI counters view empty for vertices, tasks, attempts TEZ-2924. Framework for Hadoop shims. http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js index 384a8d3..ddf6221 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js @@ -57,15 +57,14 @@ App.DagController = App.PollingController.extend(App.Helpers.DisplayHelper, { var loaders = []; var applicationId = dag.get('applicationId'); - App.Helpers.misc.removeRecord(this.store, 'appDetail', applicationId); - var appDetailLoader = this.store.find('appDetail', applicationId) + var appDetailLoader = App.Helpers.misc.loadApp(this.store, applicationId) .then(function(app){ dag.set('appDetail', app); - var appState = app.get('appState'); - if (appState) { - dag.set('yarnAppState', appState); + var status = app.get('status'); + if (status) { + dag.set('yarnAppState', status); } - dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'), app.get('appState'), app.get('finalAppStatus'))); + dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'), app.get('status'), app.get('finalStatus'))); }).catch(function(){}); App.Helpers.misc.removeRecord(this.store, 'tezApp', 'tez_' + applicationId); var tezAppLoader = this.store.find('tezApp', 'tez_' + applicationId) http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js index 3d48e37..ad9d636 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js @@ -125,44 +125,34 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C entities.forEach(function (dag) { var appId = dag.get('applicationId'); - if(appId) { - record = store.getById('appDetail', appId); - if(record && !App.Helpers.misc.isStatusInUnsuccessful(record.get('appState'))) { - store.unloadRecord(record); - } - } - }); - entities.forEach(function (dag) { - var appId = dag.get('applicationId'); - if(appId) { - store.find('appDetail', appId).then(function (app) { + if(appId && dag.get('status') === 'RUNNING') { + App.Helpers.misc.loadApp(store, appId).then(function (app) { dag.set('appDetail', app); - if (dag.get('status') === 'RUNNING') { - dag.set('status', App.Helpers.misc.getRealStatus( - dag.get('status'), - app.get('appState'), - app.get('finalAppStatus') - )); + dag.set('status', App.Helpers.misc.getRealStatus( + dag.get('status'), + app.get('status'), + app.get('finalStatus') + )); + }).catch(function(error) {}) + .finally(function () { + if(dag.get('status') === 'RUNNING') { + App.Helpers.misc.removeRecord(store, 'dagProgress', dag.get('id')); + store.find('dagProgress', dag.get('id'), { + appId: dag.get('applicationId'), + dagIdx: dag.get('idx') + }) + .then(function(dagProgressInfo) { + dag.set('progress', dagProgressInfo.get('progress')); + }) + .catch(function(error) { + error.message = "Failed to fetch dagProgress. Application Master (AM) is out of reach. Either it's down, or CORS is not enabled for YARN ResourceManager."; + Em.Logger.error(error); + var err = App.Helpers.misc.formatError(error); + var msg = 'Error code: %@, message: %@'.fmt(err.errCode, err.msg); + App.Helpers.ErrorBar.getInstance().show(msg, err.details); + }); } - }).catch(function(error) {}); - - if (dag.get('status') === 'RUNNING') { - App.Helpers.misc.removeRecord(store, 'dagProgress', dag.get('id')); - store.find('dagProgress', dag.get('id'), { - appId: dag.get('applicationId'), - dagIdx: dag.get('idx') - }) - .then(function(dagProgressInfo) { - dag.set('progress', dagProgressInfo.get('progress')); - }) - .catch(function(error) { - error.message = "Failed to fetch dagProgress. Application Master (AM) is out of reach. Either it's down, or CORS is not enabled for YARN ResourceManager."; - Em.Logger.error(error); - var err = App.Helpers.misc.formatError(error); - var msg = 'Error code: %@, message: %@'.fmt(err.errCode, err.msg); - App.Helpers.ErrorBar.getInstance().show(msg, err.details); - }); - } + }); } }); }).catch(function(error){ @@ -303,7 +293,7 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C getCellContent: function(row) { var appId = row.get('applicationId'); if(appId) { - return store.find('appDetail', appId).then(function (app) { + return App.Helpers.misc.loadApp(store, appId, true).then(function (app) { return app.get('queue'); }).catch(function(error) {}); } http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js index 0d787a9..0f11379 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js @@ -79,13 +79,12 @@ App.TaskAttemptsController = App.TablePageController.extend(App.AutoCounterColum var appDetailFetcher = that.store.find('dag', that.get('controllers.task.dagID')). then(function (dag) { - App.Helpers.misc.removeRecord(that.store, 'appDetail', dag.get('applicationId')); - return that.store.find('appDetail', dag.get('applicationId')); + return App.Helpers.misc.loadApp(that.store, dag.get('applicationId')); }). then(function(appDetail) { - var appState = appDetail.get('appState'); - if (appState) { - that.set('yarnAppState', appState); + var status = appDetail.get('status'); + if (status) { + that.set('yarnAppState', status); } }); loaders.push(appDetailFetcher); http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js index cf9b619..afb05ea 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js @@ -39,7 +39,7 @@ App.TezAppController = App.BaseController.extend(App.Helpers.DisplayHelper, App. }, pollsterControl: function () { - if(this.get('appDetail.finalAppStatus') == 'UNDEFINED' && + if(this.get('appDetail.finalStatus') == 'UNDEFINED' && this.get('pollingEnabled') && this.get('isActive')) { this.get('pollster').start(); @@ -47,7 +47,7 @@ App.TezAppController = App.BaseController.extend(App.Helpers.DisplayHelper, App. else { this.get('pollster').stop(); } - }.observes('appDetail.finalAppStatus', 'isActive', 'pollingEnabled'), + }.observes('appDetail.finalStatus', 'isActive', 'pollingEnabled'), load: function () { var tezApp = this.get('content'), @@ -56,8 +56,7 @@ App.TezAppController = App.BaseController.extend(App.Helpers.DisplayHelper, App. tezApp.reload().then(function (tezApp) { var appId = tezApp.get('appId'); if(!appId) return tezApp; - App.Helpers.misc.removeRecord(store, 'appDetail', appId); - return store.find('appDetail', appId).then(function (appDetails){ + return App.Helpers.misc.loadApp(store, appId).then(function (appDetails){ tezApp.set('appDetail', appDetails); return tezApp; }); http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js index 2a9f9ae..a4dc372 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js @@ -32,8 +32,7 @@ App.TezAppIndexController = App.PollingController.extend(App.ModelRefreshMixin, tezApp.reload().then(function (tezApp) { var appId = tezApp.get('appId'); if(!appId) return tezApp; - App.Helpers.misc.removeRecord(store, 'appDetail', appId); - return store.find('appDetail', appId).then(function (appDetails){ + return App.Helpers.misc.loadApp(store, appId).then(function (appDetails){ tezApp.set('appDetail', appDetails); return tezApp; }); @@ -50,6 +49,6 @@ App.TezAppIndexController = App.PollingController.extend(App.ModelRefreshMixin, }.property('appDetail.user', 'user'), iconStatus: function() { - return App.Helpers.misc.getStatusClassForEntity(this.get('model.appDetail.finalAppStatus')); - }.property('id', 'appDetail.finalAppStatus'), + return App.Helpers.misc.getStatusClassForEntity(this.get('model.appDetail.finalStatus')); + }.property('id', 'appDetail.finalStatus'), }); http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js index f33eb99..feb4983 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js @@ -82,14 +82,13 @@ App.VertexController = App.PollingController.extend(App.Helpers.DisplayHelper, A loaders.push(progressLoader); } - App.Helpers.misc.removeRecord(that.store, 'appDetail', applicationId); - var appDetailFetcher = that.store.find('appDetail', applicationId).then(function(appDetail) { - var appState = appDetail.get('appState'); - if (appState) { - vertex.set('yarnAppState', appState); + var appDetailFetcher = App.Helpers.misc.loadApp(that.store, applicationId).then(function(appDetail) { + var status = appDetail.get('status'); + if (status) { + vertex.set('yarnAppState', status); } - vertex.set('status', App.Helpers.misc.getRealStatus(vertex.get('status'), appDetail.get('appState'), - appDetail.get('finalAppStatus'))); + vertex.set('status', App.Helpers.misc.getRealStatus(vertex.get('status'), appDetail.get('status'), + appDetail.get('finalStatus'))); }).catch(function(){}); loaders.push(appDetailFetcher); http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/helpers/misc.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js index e0246f4..b6fd690 100644 --- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js +++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js @@ -789,6 +789,26 @@ App.Helpers.misc = { defaultQueryParamsConfig: { refreshModel: true, replace: true + }, + + /** + * Load app details form RM if available, else load from ATS if AHS is enabled + * @param store {Store} + * @param appId {String} + * @param useCache {Boolean} + */ + loadApp: function (store, appId, useCache) { + if(!useCache) { + App.Helpers.misc.removeRecord(store, 'appDetail', appId); + App.Helpers.misc.removeRecord(store, 'clusterApp', appId); + } + + return store.find('clusterApp', appId).catch(function () { + return store.find('appDetail', appId); + }).catch(function (error) { + error.message = "Couldn't get details of application %@. RM is not reachable, and history service is not enabled.".fmt(appId); + throw error; + }); } } http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js b/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js index d54c8b9..b4a2062 100644 --- a/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js +++ b/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js @@ -482,9 +482,9 @@ var timelineJsonToAppDetailMap = { finishedTime: 'finishedTime', submittedTime: 'submittedTime', - appState: 'appState', + status: 'appState', - finalAppStatus: 'finalAppStatus', + finalStatus: 'finalAppStatus', diagnostics: 'otherinfo.diagnostics', }; @@ -637,7 +637,18 @@ App.ClusterAppSerializer = App.TimelineSerializer.extend({ map: { id: 'id', status: 'state', - finalStatus: 'finalStatus' + finalStatus: 'finalStatus', + + name: 'name', + queue: 'queue', + user: 'user', + type: 'type', + + startedTime: 'startedTime', + elapsedTime: 'elapsedTime', + finishedTime: 'finishedTime', + + progress: 'progress' }, _normalizeSingleDagPayload: function(rawPayload) { http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/models/dag.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/models/dag.js b/tez-ui/src/main/webapp/app/scripts/models/dag.js index 3442433..ff4a388 100644 --- a/tez-ui/src/main/webapp/app/scripts/models/dag.js +++ b/tez-ui/src/main/webapp/app/scripts/models/dag.js @@ -48,7 +48,7 @@ App.Dag = App.AbstractEntity.extend({ }.property('id'), tezApp: DS.belongsTo('tezApp'), - appDetail: DS.belongsTo('appDetail'), + appDetail: DS.attr('object'), progress: DS.attr('number'), @@ -319,8 +319,8 @@ App.AppDetail = App.AbstractEntity.extend({ queue: DS.attr('string'), type: DS.attr('string'), - appState: DS.attr('string'), - finalAppStatus: DS.attr('string'), + status: DS.attr('string'), + finalStatus: DS.attr('string'), progress: DS.attr('string'), startedTime: DS.attr('number'), @@ -339,7 +339,7 @@ App.TezApp = App.AbstractEntity.extend({ startedTime: DS.attr('number'), - appDetail: DS.belongsTo('appDetail', { async: true }), + appDetail: DS.attr('object'), dags: DS.hasMany('dag', { async: true }), configs: DS.hasMany('kVData', { async: false }), @@ -353,6 +353,18 @@ App.ClusterApp = App.AbstractEntity.extend({ status: DS.attr('string'), finalStatus: DS.attr('string'), + user: DS.attr('string'), + name: DS.attr('string'), + queue: DS.attr('string'), + type: DS.attr('string'), + + startedTime: DS.attr('number'), + elapsedTime: DS.attr('number'), + finishedTime: DS.attr('number'), + submittedTime: DS.attr('number'), + + progress: DS.attr('number'), + isComplete: function () { var status = this.get('status'); return status == 'FINISHED' || status == 'FAILED' || status == 'KILLED'; http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/scripts/router.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/router.js b/tez-ui/src/main/webapp/app/scripts/router.js index a575dea..2478eaf 100644 --- a/tez-ui/src/main/webapp/app/scripts/router.js +++ b/tez-ui/src/main/webapp/app/scripts/router.js @@ -271,7 +271,7 @@ App.TezAppRoute = App.BaseRoute.extend({ var store = this.store; return store.find('tezApp', 'tez_' + params.app_id).then(function (tezApp){ if(!tezApp.get('appId')) return tezApp; - return store.find('appDetail', tezApp.get('appId')).then(function (appDetails){ + return App.Helpers.misc.loadApp(store, tezApp.get('appId')).then(function (appDetails){ tezApp.set('appDetail', appDetails); return tezApp; }).catch(function() { http://git-wip-us.apache.org/repos/asf/tez/blob/98706f27/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs b/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs index 3930d7e..ff87fbc 100644 --- a/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs +++ b/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs @@ -74,14 +74,14 @@ <tr> <td>Application State</td> <td> - {{appDetail.appState}} + {{appDetail.status}} </td> </tr> <tr> <td>Final Application Status</td> <td> - {{#if appDetail.appState}} - <i {{bind-attr class=':task-status iconStatus'}}></i> {{appDetail.finalAppStatus}} + {{#if appDetail.status}} + <i {{bind-attr class=':task-status iconStatus'}}></i> {{appDetail.finalStatus}} {{/if}} </td> </tr>
