TEZ-2159. Tez UI: download timeline data for offline use. (Prakash Ramachandran via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/62a348ce Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/62a348ce Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/62a348ce Branch: refs/heads/TEZ-2003 Commit: 62a348ceae36939b0c5818ee626e28a5a0ae70e3 Parents: f2d560c Author: Hitesh Shah <[email protected]> Authored: Tue Apr 7 11:48:14 2015 -0700 Committer: Hitesh Shah <[email protected]> Committed: Tue Apr 7 11:48:14 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 +- tez-ui/src/main/resources/META-INF/LICENSE.txt | 2 + tez-ui/src/main/webapp/Gruntfile.js | 22 ++ tez-ui/src/main/webapp/app/index.html | 2 + .../scripts/controllers/dag_index_controller.js | 37 +++ .../src/main/webapp/app/scripts/helpers/io.js | 262 +++++++++++++++++++ .../src/main/webapp/app/scripts/helpers/misc.js | 123 +++++++++ .../src/main/webapp/app/templates/dag/index.hbs | 5 + tez-ui/src/main/webapp/bower.json | 4 +- 9 files changed, 457 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 11b843d..0eb02a7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,8 +8,8 @@ INCOMPATIBLE CHANGES TEZ-2176. Move all logging to slf4j. (commons-logging jar no longer part of Tez tar) TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly - ALL CHANGES: + TEZ-2159. Tez UI: download timeline data for offline use. TEZ-2269. DAGAppMaster becomes unresponsive (post TEZ-2149). TEZ-2243. documentation should explicitly specify protobuf 2.5.0. TEZ-2232. Allow setParallelism to be called multiple times before tasks get http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/resources/META-INF/LICENSE.txt ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/resources/META-INF/LICENSE.txt b/tez-ui/src/main/resources/META-INF/LICENSE.txt index 6f90d29..b7c8a86 100644 --- a/tez-ui/src/main/resources/META-INF/LICENSE.txt +++ b/tez-ui/src/main/resources/META-INF/LICENSE.txt @@ -226,6 +226,7 @@ The Apache TEZ tez-ui bundles the following files under the MIT License: - jquery-mousewheel v3.1.12 (https://github.com/jquery/jquery-mousewheel) - Copyright 2006, 2014 jQuery Foundation and other contributors, https://jquery.org/ - jquery-ui v1.11 (http://jqueryui.com/) - Copyright 2014 jQuery Foundation and other contributors - moment v2.8.4 (http://momentjs.com/) - authors : Tim Wood, Iskren Chernev, Moment.js contributors + - FileSaver.js master branch #24b303f49213b905ec9062b708f7cd43d56a5dde (https://github.com/eligrey/FileSaver.js) authors : Eli Grey. All rights reserved. @@ -256,6 +257,7 @@ The Apache Tez tez-ui bundles the following files under BSD licenses: (3-clause BSD license) - D3 v3.4.11 (http://d3js.org/) - Copyright (c) 2010-2014, Michael Bostock - ember-table v0.2.2 (https://github.com/Addepar/ember-table) - Copyright © 2012 Addepar, Inc. All Rights Reserved. + - zip.js master branch #bfd76c66293305faaf9fcbb65b5ff7fe2dbe621a (https://github.com/gildas-lormeau/zip.js) All rights reserved. http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/Gruntfile.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/Gruntfile.js b/tez-ui/src/main/webapp/Gruntfile.js index 95e57da..98f62b1 100644 --- a/tez-ui/src/main/webapp/Gruntfile.js +++ b/tez-ui/src/main/webapp/Gruntfile.js @@ -281,6 +281,17 @@ module.exports = function (grunt) { { expand: true, flatten: true, + cwd: '<%= yeoman.app %>', + dest: '<%= yeoman.dist %>/scripts/zip.js', + src: [ + 'bower_components/zip.js/WebContent/z-worker.js', + 'bower_components/zip.js/WebContent/inflate.js', + 'bower_components/zip.js/WebContent/deflate.js', + ] + }, + { + expand: true, + flatten: true, src: '<%= yeoman.app %>/bower_components/jquery-ui/themes/smoothness/images/*', dest: '<%= yeoman.dist %>/styles/images/' }, @@ -326,6 +337,17 @@ module.exports = function (grunt) { }, { expand: true, + flatten: true, + cwd: '<%= yeoman.app %>', + dest: '.tmp/scripts/zip.js', + src: [ + 'bower_components/zip.js/WebContent/z-worker.js', + 'bower_components/zip.js/WebContent/inflate.js', + 'bower_components/zip.js/WebContent/deflate.js', + ] + }, + { + expand: true, flatten: false, cwd: '<%= yeoman.app %>', src: 'bower_components/**', http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/index.html ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/index.html b/tez-ui/src/main/webapp/app/index.html index b2d1ac3..f87b2a6 100644 --- a/tez-ui/src/main/webapp/app/index.html +++ b/tez-ui/src/main/webapp/app/index.html @@ -60,6 +60,8 @@ <script src="bower_components/ember-table/dist/ember-table.js"></script> <script src="bower_components/ember-addons.bs_for_ember/dist/js/bs-nav.min.js"></script> <script src="bower_components/d3/d3.js"></script> + <script src="bower_components/zip.js/WebContent/zip.js"></script> + <script src="bower_components/FileSaver.js/FileSaver.min.js"></script> <!-- endbuild --> <!-- build:js(.tmp) scripts/templates.js --> http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js index 7576149..416f500 100644 --- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js +++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js @@ -22,6 +22,43 @@ App.DagIndexController = Em.ObjectController.extend(App.ModelRefreshMixin, { needs: 'dag', + actions: { + downloadDagJson: function() { + var dagID = this.get('id'); + var downloader = App.Helpers.misc.downloadDAG(this.get('id'), { + batchSize: 500, + onSuccess: function() { + Bootstrap.ModalManager.close('downloadModal'); + }, + onFailure: function() { + $('#modalMessage').html('<i class="fa fa-lg fa-exclamation-circle margin-small-horizontal" ' + + 'style="color:red"></i> Error downloading data'); + } + }); + this.set('tmpDownloader', downloader); + var modalDialogView = Ember.View.extend({ + template: Em.Handlebars.compile( + '<p id="modalMessage"><i class="fa fa-lg fa-spinner fa-spin margin-small-horizontal" ' + + 'style="color:green"></i>Downloading data for dag %@</p>'.fmt(dagID) + ) + }); + var buttons = [ + Ember.Object.create({title: 'Cancel', dismiss: 'modal', clicked: 'cancelDownload'}) + ]; + Bootstrap.ModalManager.open('downloadModal', 'Download data', + modalDialogView, buttons, this); + }, + + cancelDownload: function() { + var currentDownloader = this.get('tmpDownloader'); + if (!!currentDownloader) { + currentDownloader.cancel(); + } + this.set('tmpDownloader', undefined); + } + + }, + load: function () { var dag = this.get('controllers.dag.model'), controller = this.get('controllers.dag'), http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/scripts/helpers/io.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/io.js b/tez-ui/src/main/webapp/app/scripts/helpers/io.js new file mode 100644 index 0000000..e959e0a --- /dev/null +++ b/tez-ui/src/main/webapp/app/scripts/helpers/io.js @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +zip.workerScriptsPath = "scripts/zip.js/"; + +App.Helpers.io = { + /* Allow queuing of downloads and then get a callback once all the downloads are done. + * sample usage. + * var downloader = App.Helpers.io.fileDownloader(); + * downloader.queueItem({ + * url: 'http://....', + * onItemFetched: function(data, context) {...}, + * context: {}, // context object gets passed back to the callback + * }); + * downloader.queueItem({...}); //queue in other items + * downloader.finish(); // once all items are queued. items can be queued from + * // callbacks too. in that case the finish should be called + * // once all items are queued. + * downloader.then(successCallback).catch(failurecallback).finally(callback) + */ + fileDownloader: function(options) { + var itemList = [], + opts = options || {}, + numParallel = opts.numParallel || 5, + hasMoreInputs = true, + inProgress = 0, + hasFailed = false, + pendingRequests = {}, + pendingRequestID = 0, + failureReason = 'Unknown', + deferredPromise = Em.RSVP.defer(); + + function checkForCompletion() { + if (hasFailed) { + if (inProgress == 0) { + deferredPromise.reject("FOOBAR"); + } + return; + } + + if (hasMoreInputs || itemList.length > 0 || inProgress > 0) { + return; + } + + deferredPromise.resolve(); + } + + function getRequestId() { + return "req_" + pendingRequestID++; + } + + function abortPendingRequests() { + $.each(pendingRequests, function(idx, val) { + try { + val.abort("abort"); + } catch(e) {} + }); + } + + function markFailed(reason) { + if (!hasFailed) { + hasFailed = true; + failureReason = reason; + abortPendingRequests(); + } + } + + function processNext() { + if (inProgress >= numParallel) { + Em.Logger.debug("delaying download as %@ of %@ is in progress".fmt(inProgress, numParallel)); + return; + } + + if (itemList.length < 1) { + Em.Logger.debug("no items to download"); + checkForCompletion(); + return; + } + + inProgress++; + Em.Logger.debug("starting download %@".fmt(inProgress)); + var item = itemList.shift(); + + var xhr = $.getJSON(item.url); + var reqID = getRequestId(); + pendingRequests[reqID] = xhr; + + xhr.done(function(data, statusText, xhr) { + delete pendingRequests[reqID]; + + if ($.isFunction(item.onItemFetched)) { + try { + item.onItemFetched(data, item.context); + } catch (e) { + markFailed("invalid data"); + inProgress--; + checkForCompletion(); + return; + } + } + + inProgress--; + processNext(); + }).fail(function(xhr, statusText, errorObject) { + delete pendingRequests[reqID]; + markFailed(statusText); + inProgress--; + checkForCompletion(); + }); + } + + return DS.PromiseObject.create({ + promise: deferredPromise.promise, + + queueItems: function(options) { + options.forEach(this.queueItem); + }, + + queueItem: function(option) { + itemList.push(option); + processNext(); + }, + + finish: function() { + hasMoreInputs = false; + checkForCompletion(); + }, + + cancel: function() { + markFailed("User cancelled"); + checkForCompletion(); + } + }); + }, + + + /* + * allows to zip files and download that. + * usage: + * zipHelper = App.Helpers.io.zipHelper({ + * onProgress: function(filename, current, total) { ...}, + * onAdd: function(filename) {...} + * }); + * zipHelper.addFile({name: filenameinsidezip, data: data); + * // add all files + * once all files are added call the close + * zipHelper.close(); // or .abort to abort zip + * zipHelper.then(function(zippedBlob) { + * saveAs(filename, zippedBlob); + * }).catch(failureCallback); + */ + zipHelper: function(options) { + var opts = options || {}, + zipFileEntry, + zipWriter, + completion = Em.RSVP.defer(), + fileList = [], + completed = 0, + currentIdx = -1, + numFiles = 0, + hasMoreInputs = true, + inProgress = false, + hasFailed = false; + + zip.createWriter(new zip.BlobWriter(), function(writer) { + zipWriter = writer; + checkForCompletion(); + nextFile(); + }); + + function checkForCompletion() { + if (hasFailed) { + if (zipWriter) { + Em.Logger.debug("aborting zipping. closing file."); + zipWriter.close(completion.reject); + zipWriter = null; + } + } else { + if (!hasMoreInputs && numFiles == completed) { + Em.Logger.debug("completed zipping. closing file."); + zipWriter.close(completion.resolve); + } + } + } + + function onProgress(current, total) { + if ($.isFunction(opts.onProgress)) { + opts.onProgress(fileList[currentIdx].name, current, total); + } + } + + function onAdd(filename) { + if ($.isFunction(opts.onAdd)) { + opts.onAdd(filename); + } + } + + function nextFile() { + if (hasFailed || completed == numFiles || inProgress) { + return; + } + + currentIdx++; + var file = fileList[currentIdx]; + inProgress = true; + onAdd(file.name); + zipWriter.add(file.name, new zip.TextReader(file.data), function() { + completed++; + inProgress = false; + if (currentIdx < numFiles - 1) { + nextFile(); + } + checkForCompletion(); + }, onProgress); + } + + return DS.PromiseObject.create({ + addFiles: function(files) { + files.forEach(this.addFile); + }, + + addFile: function(file) { + if (hasFailed) { + Em.Logger.debug("Skipping add of file %@ as zip has been aborted".fmt(file.name)); + return; + } + numFiles++; + fileList.push(file); + if (zipWriter) { + Em.Logger.debug("addinng file from addFile: " + file.name); + nextFile(); + } + }, + + close: function() { + hasMoreInputs = false; + checkForCompletion(); + }, + + promise: completion.promise, + + abort: function() { + hasFailed = true; + this.close(); + } + }); + } +}; http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/scripts/helpers/misc.js ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js index 608f60e..fdd69bd 100644 --- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js +++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js @@ -270,6 +270,129 @@ App.Helpers.misc = { } }, + downloadDAG: function(dagID, options) { + var opts = options || {}, + batchSize = opts.batchSize || 1000, + baseurl = '%@/%@'.fmt(App.env.timelineBaseUrl, App.Configs.restNamespace.timeline), + itemsToDownload = [ + { + url: getUrl('TEZ_DAG_ID', dagID), + context: { name: 'dag', type: 'TEZ_DAG_ID' }, + onItemFetched: processSingleItem + }, + { + url: getUrl('TEZ_VERTEX_ID', dagID), + context: { name: 'vertices', type: 'TEZ_VERTEX_ID', part: 0 }, + onItemFetched: processMultipleItems + }, + { + url: getUrl('TEZ_TASK_ID', dagID), + context: { name: 'tasks', type: 'TEZ_TASK_ID', part: 0 }, + onItemFetched: processMultipleItems + }, + { + url: getUrl('TEZ_TASK_ATTEMPT_ID', dagID), + context: { name: 'task_attempts', type: 'TEZ_TASK_ATTEMPT_ID', part: 0 }, + onItemFetched: processMultipleItems + } + ], + numItemTypesToDownload = itemsToDownload.length, + downloader = App.Helpers.io.fileDownloader(), + zipHelper = App.Helpers.io.zipHelper({ + onProgress: function(filename, current, total) { + Em.Logger.debug('%@: %@ of %@'.fmt(filename, current, total)); + }, + onAdd: function(filename) { + Em.Logger.debug('adding %@ to Zip'.fmt(filename)); + } + }); + + function getUrl(type, dagID, fromID) { + var url; + if (type == 'TEZ_DAG_ID') { + url = '%@/%@/%@'.fmt(baseurl, type, dagID); + } else { + url = '%@/%@?primaryFilter=TEZ_DAG_ID:%@&limit=%@'.fmt(baseurl, type, dagID, batchSize + 1); + if (!!fromID) { + url = '%@&fromId=%@'.fmt(url, fromID); + } + } + return url; + } + + function checkIfAllDownloaded() { + numItemTypesToDownload--; + if (numItemTypesToDownload == 0) { + downloader.finish(); + } + } + + function processSingleItem(data, context) { + var obj = {}; + obj[context.name] = data; + + zipHelper.addFile({name: '%@.json'.fmt(context.name), data: JSON.stringify(obj, null, 2)}); + checkIfAllDownloaded(); + } + + function processMultipleItems(data, context) { + var obj = {}; + var nextBatchStart = undefined; + + if (!$.isArray(data.entities)) { + throw "invalid data"; + } + + // need to handle no more entries , zero entries + if (data.entities.length > batchSize) { + nextBatchStart = data.entities.pop().entity; + } + obj[context.name] = data.entities; + + zipHelper.addFile({name: '%@_part_%@.json'.fmt(context.name, context.part), data: JSON.stringify(obj, null, 2)}); + + if (!!nextBatchStart) { + context.part++; + downloader.queueItem({ + url: getUrl(context.type, dagID, nextBatchStart), + context: context, + onItemFetched: processMultipleItems + }); + } else { + checkIfAllDownloaded(); + } + } + + downloader.queueItems(itemsToDownload); + + downloader.then(function() { + Em.Logger.info('Finished download'); + zipHelper.close(); + }).catch(function() { + Em.Logger.error('Failed to download'); + zipHelper.abort(); + }); + + var that = this; + zipHelper.then(function(zippedBlob) { + saveAs(zippedBlob, '%@.zip'.fmt(dagID)); + if ($.isFunction(opts.onSuccess)) { + opts.onSuccess(); + } + }).catch(function() { + Em.Logger.error('zip Failed'); + if ($.isFunction(opts.onFailure)) { + opts.onFailure(); + } + }); + + return { + cancel: function() { + downloader.cancel(); + } + } + }, + dagStatusUIOptions: [ { label: 'All', id: null }, { label: 'Submitted', id: 'SUBMITTED' }, http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/app/templates/dag/index.hbs ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/app/templates/dag/index.hbs b/tez-ui/src/main/webapp/app/templates/dag/index.hbs index c17d87f..e7c6264 100644 --- a/tez-ui/src/main/webapp/app/templates/dag/index.hbs +++ b/tez-ui/src/main/webapp/app/templates/dag/index.hbs @@ -33,6 +33,11 @@ </thead> <tbody> <tr> + <td colspan="2"> + {{bs-button icon="fa fa-download" title="Download data" type="info" clicked="downloadDagJson"}} + </td> + </tr> + <tr> <td>{{t 'common.applicationId'}}</td> <td> {{#if controllers.dag.enableAppIdLink}} http://git-wip-us.apache.org/repos/asf/tez/blob/62a348ce/tez-ui/src/main/webapp/bower.json ---------------------------------------------------------------------- diff --git a/tez-ui/src/main/webapp/bower.json b/tez-ui/src/main/webapp/bower.json index b3d8805..54f6f57 100644 --- a/tez-ui/src/main/webapp/bower.json +++ b/tez-ui/src/main/webapp/bower.json @@ -13,7 +13,9 @@ "d3": "3.4.11", "ember-addons.bs_for_ember": "~0.7.0", "ember-table": "~0.2.4", - "font-awesome":"4.2.0" + "font-awesome":"4.2.0", + "FileSaver.js": "https://github.com/eligrey/FileSaver.js.git#24b303f49213b905ec9062b708f7cd43d56a5dde";, + "zip.js": "https://github.com/gildas-lormeau/zip.js.git#bfd76c66293305faaf9fcbb65b5ff7fe2dbe621a"; }, "resolutions": { "jquery": "1.10.2",
