AMBARI-19628. Hive View 2.0: Ability to view and create table and column statistics. (dipayanb)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/22d4e181 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/22d4e181 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/22d4e181 Branch: refs/heads/branch-dev-patch-upgrade Commit: 22d4e18168935d9a9400591ebcd1a05c2b2ebf7d Parents: f88aca8 Author: Dipayan Bhowmick <[email protected]> Authored: Fri Jan 27 12:03:24 2017 +0530 Committer: Dipayan Bhowmick <[email protected]> Committed: Fri Jan 27 12:03:56 2017 +0530 ---------------------------------------------------------------------- .../src/main/resources/ui/app/adapters/table.js | 27 +++- .../ui/app/components/table-statistics.js | 120 +++++++++++++++ .../main/resources/ui/app/models/table-info.js | 1 + .../routes/databases/database/tables/table.js | 4 + .../src/main/resources/ui/app/services/jobs.js | 10 +- .../resources/ui/app/services/stats-service.js | 76 +++++++++ .../src/main/resources/ui/app/styles/app.scss | 23 +-- .../templates/components/table-statistics.hbs | 153 +++++++++++++++++++ .../app/templates/databases/database/tables.hbs | 2 +- .../databases/database/tables/table/stats.hbs | 4 +- 10 files changed, 399 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/adapters/table.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/adapters/table.js b/contrib/views/hive20/src/main/resources/ui/app/adapters/table.js index 9a4692d..e878899 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/adapters/table.js +++ b/contrib/views/hive20/src/main/resources/ui/app/adapters/table.js @@ -22,14 +22,14 @@ import DDLAdapter from './ddl'; export default DDLAdapter.extend({ buildURL(modelName, id, snapshot, requestType, query) { // Check if the query is to find all tables for a particular database - if(Ember.isEmpty(id) && (requestType === 'query' || requestType == 'queryRecord')) { + if (Ember.isEmpty(id) && (requestType === 'query' || requestType == 'queryRecord')) { let dbId = query.databaseId; let tableName = query.tableName; let origFindAllUrl = this._super(...arguments); let prefix = origFindAllUrl.substr(0, origFindAllUrl.lastIndexOf("/")); delete query.databaseId; delete query.tableName; - if(Ember.isEmpty(tableName)) { + if (Ember.isEmpty(tableName)) { return `${prefix}/databases/${dbId}/tables`; } else { return `${prefix}/databases/${dbId}/tables/${tableName}`; @@ -40,12 +40,29 @@ export default DDLAdapter.extend({ createTable(tableMetaInfo) { - let postURL = this.buildURL('table', null, null, 'query', {databaseId: tableMetaInfo.database}); - return this.ajax(postURL, 'POST', { data: {tableInfo: tableMetaInfo} }); + let postURL = this.buildURL('table', null, null, 'query', { databaseId: tableMetaInfo.database }); + return this.ajax(postURL, 'POST', { data: { tableInfo: tableMetaInfo } }); }, deleteTable(database, tableName) { - let deletURL = this.buildURL('table', null, null, 'query', {databaseId: database, tableName: tableName}); + let deletURL = this.buildURL('table', null, null, 'query', { databaseId: database, tableName: tableName }); return this.ajax(deletURL, 'DELETE'); + }, + + analyseTable(databaseName, tableName, withColumns = false) { + let analyseUrl = this.buildURL('table', null, null, 'query', { databaseId: databaseName, tableName: tableName }) + + '/analyze' + + (withColumns ? '?analyze_columns=true' : ''); + return this.ajax(analyseUrl, 'PUT'); + }, + + generateColumnStats(databaseName, tableName, columnName) { + let url = this.buildURL('table', null, null, 'query', {databaseId: databaseName, tableName: tableName}) + `/column/${columnName}/stats`; + return this.ajax(url, 'GET'); + }, + + fetchColumnStats(databaseName, tableName, columnName, jobId) { + let url = this.buildURL('table', null, null, 'query', {databaseId: databaseName, tableName: tableName}) + `/column/${columnName}/fetch_stats?job_id=${jobId}`; + return this.ajax(url, 'GET'); } }); http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js b/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js new file mode 100644 index 0000000..d53a41f --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; + +export default Ember.Component.extend({ + statsService: Ember.inject.service(), + + analyseWithStatistics: false, + + tableStats: Ember.computed.oneWay('table.tableStats'), + tableStatisticsEnabled: Ember.computed.oneWay('table.tableStats.isTableStatsEnabled'), + + columnStatsAccurate: Ember.computed('table.tableStats.columnStatsAccurate', function () { + let columnStatsJson = this.get('table.tableStats.columnStatsAccurate'); + return JSON.parse(columnStatsJson.replace(/\\\"/g, '"')); + }), + + columnsWithStatistics: Ember.computed('columnStatsAccurate', function () { + let stats = this.get('columnStatsAccurate.COLUMN_STATS'); + return !stats ? [] : Object.keys(stats); + }), + + columns: Ember.computed('table.columns', 'columnsWithStatistics', function () { + let cols = this.get('table.columns'); + let colsWithStatistics = this.get('columnsWithStatistics'); + return cols.map((col) => { + let copy = Ember.Object.create(col); + copy.set('hasStatistics', colsWithStatistics.contains(copy.name)); + copy.set('isFetchingStats', false); + copy.set('statsError', false); + copy.set('showStats', true); + return copy; + }); + }), + + allColumnsHasStatistics: Ember.computed('table.columns', 'columnsWithStatistics', function () { + let colsNames = this.get('table.columns').getEach('name'); + let colsWithStatistics = this.get('columnsWithStatistics'); + + let colsNotIn = colsNames.filter((item) => !colsWithStatistics.contains(item)); + return colsNotIn.length === 0; + }), + + performTableAnalysis(withColumns = false) { + const tableName = this.get('table.table'); + const databaseName = this.get('table.database'); + + let title = `Analyse table` + (withColumns ? ' for columns' : ''); + this.set('analyseTitle', title); + this.set('analyseMessage', `Submitting job to generate statistics for table '${tableName}'`); + + this.set('showAnalyseModal', true); + + this.get('statsService').generateStatistics(databaseName, tableName, withColumns) + .then((job) => { + this.set('analyseMessage', 'Waiting for the job to complete'); + return this.get('statsService').waitForStatsGenerationToComplete(job); + }).then(() => { + this.set('analyseMessage', 'Finished analysing table for statistics'); + Ember.run.later(() => this.closeAndRefresh(), 2 * 1000); + }).catch((err) => { + this.set('analyseMessage', 'Job failed for analysing statistics of table'); + Ember.run.later(() => this.closeAndRefresh(), 2 * 1000); + }); + }, + + fetchColumnStats(column) { + const tableName = this.get('table.table'); + const databaseName = this.get('table.database'); + + column.set('isFetchingStats', true); + + this.get('statsService').generateColumnStatistics(databaseName, tableName, column.name).then((job) => { + return this.get('statsService').waitForStatsGenerationToComplete(job, false); + }).then((job) => { + return this.get('statsService').fetchColumnStatsResult(databaseName, tableName, column.name, job); + }).then((data) => { + column.set('isFetchingStats', false); + column.set('stats', data); + }).catch((err) => { + column.set('isFetchingStats', false); + column.set('statsError', true); + }); + }, + + closeAndRefresh() { + this.set('showAnalyseModal', false); + this.sendAction('refresh'); + }, + + actions: { + analyseTable() { + this.performTableAnalysis(this.get('analyseWithStatistics')); + }, + + fetchStats(column) { + this.fetchColumnStats(column); + }, + + toggleShowStats(column) { + column.toggleProperty('showStats'); + } + } +}); http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/models/table-info.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/models/table-info.js b/contrib/views/hive20/src/main/resources/ui/app/models/table-info.js index 85306b6..ea51ab6 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/models/table-info.js +++ b/contrib/views/hive20/src/main/resources/ui/app/models/table-info.js @@ -25,6 +25,7 @@ export default DS.Model.extend({ ddl: DS.attr('string'), partitionInfo: DS.attr(), detailedInfo: DS.attr(), + tableStats: DS.attr(), storageInfo: DS.attr(), viewInfo: DS.attr() }); http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/table.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/table.js b/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/table.js index 88f1e7e..1066bc1 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/table.js +++ b/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/table.js @@ -48,6 +48,10 @@ export default Ember.Route.extend({ editTable(table) { console.log("Edit table"); + }, + + refreshTableInfo() { + this.refresh(); } }, http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/services/jobs.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/services/jobs.js b/contrib/views/hive20/src/main/resources/ui/app/services/jobs.js index ca058f2..5d7ce77 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/services/jobs.js +++ b/contrib/views/hive20/src/main/resources/ui/app/services/jobs.js @@ -26,18 +26,22 @@ export default Ember.Service.extend({ }) }, - waitForJobToComplete(jobId, after) { + waitForJobToComplete(jobId, after, fetchDummyResult = true) { + console.log() return new Ember.RSVP.Promise((resolve, reject) => { Ember.run.later(() => { - this.get('store').findRecord('job', jobId, {reload: true}) + this.get('store').findRecord('job', jobId, { reload: true }) .then((job) => { let status = job.get('status').toLowerCase(); if (status === 'succeeded') { + if (fetchDummyResult) { + this._fetchDummyResult(jobId); + } resolve(status); } else if (status === 'error') { reject(status) } else { - resolve(this.waitForJobToComplete(jobId, after)); + resolve(this.waitForJobToComplete(jobId, after, fetchDummyResult)); } }, (error) => { reject(error); http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js b/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js new file mode 100644 index 0000000..bb3ed3e --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; + +const columnStatsKeys = [ + {dataKey: 'min', label: 'MIN'}, + {dataKey: 'max', label: 'MAX'}, + {dataKey: 'numNulls', label: 'NUMBER OF NULLS'}, + {dataKey: 'distinctCount', label: 'DISTINCT COUNT'}, + {dataKey: 'avgColLen', label: 'AVERAGE COLUMN LENGTH'}, + {dataKey: 'maxColLen', label: 'MAX COLUMN LENGTH'}, + {dataKey: 'numTrues', label: 'NUMBER OF TRUE'}, + {dataKey: 'numFalse', label: 'NUMBER OF FALSE'}, + ]; + +export default Ember.Service.extend({ + jobs: Ember.inject.service(), + store: Ember.inject.service(), + + generateStatistics(databaseName, tableName, withColumns = false) { + return new Promise((resolve, reject) => { + this.get('store').adapterFor('table').analyseTable(databaseName, tableName, withColumns).then((data) => { + this.get('store').pushPayload(data); + resolve(this.get('store').peekRecord('job', data.job.id)); + }, (err) => { + reject(err); + }); + }); + }, + + generateColumnStatistics(databaseName, tableName, columnName) { + return new Promise((resolve, reject) => { + this.get('store').adapterFor('table').generateColumnStats(databaseName, tableName, columnName).then((data) => { + this.get('store').pushPayload(data); + resolve(this.get('store').peekRecord('job', data.job.id)); + }, (err) => { + reject(err); + }); + }); + }, + + waitForStatsGenerationToComplete(job, fetchDummyResult = true) { + return new Promise((resolve, reject) => { + this.get('jobs').waitForJobToComplete(job.get('id'), 5 * 1000, fetchDummyResult).then((data) => { + resolve(job); + }, (err) => { + reject(err); + }); + }); + }, + + fetchColumnStatsResult(databaseName, tableName, columnName, job) { + return this.get('store').adapterFor('table').fetchColumnStats(databaseName, tableName, columnName, job.get('id')).then((data) => { + let columnStats = data.columnStats; + return columnStatsKeys.map((item) => { + return {label: item.label, value: columnStats[item.dataKey]}; + }); + }); + } +}); http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss b/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss index 5ae65d1..e178222 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss +++ b/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss @@ -194,6 +194,10 @@ $table-info-background: lighten($body-bg, 10%); } } } + + .alert { + margin: 0; + } } .table-name-input { @@ -209,13 +213,7 @@ pre { } } -.dipayan { - .CodeMirror { - height: 100vh; - } -} - -.dipayan123 { +.scroll-fix { background-color: lighten($body-bg, 5%); height: calc(100vh - 180px); overflow-y: scroll; @@ -810,10 +808,12 @@ pre { padding-bottom: 25px; } -.authorizations { - &.alert { - margin: 0; - } +.stats-section { + margin-top: 20px; +} + +.col-stats-details { + padding-top: 10px; } @@ -825,3 +825,4 @@ pre { + http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs b/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs new file mode 100644 index 0000000..0ee3b13 --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs @@ -0,0 +1,153 @@ +{{! +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +}} + +<div class="row"> + <div class="alert"> + <p class="lead"> + {{fa-icon "line-chart" size=2}} STATISTICS + <div class="pull-right"> + <button class="btn btn-success" + {{action "analyseTable"}}>{{fa-icon "location-arrow"}} {{#if (not tableStatisticsEnabled)}} + Compute {{else}} Recompute {{/if}}</button> + + <label> + {{input type="checkbox" checked=analyseWithStatistics}} + <small>include columns</small> + </label> + </div> + </p> + + </div> +</div> +<div class="row"> + {{#if tableStatisticsEnabled}} + <div class="stats-section"> + <p><strong>TABLE STATISTICS</strong></p> + <table class="table table-bordered table-hover"> + <thead> + <tr> + <td width="50%">STATS NAME</td> + <td width="50%">VALUE</td> + </tr> + </thead> + <tbody> + <tr> + <td>Number of Files</td> + <td>{{tableStats.numFiles}}</td> + </tr> + <tr> + <td>Raw Data Size</td> + <td>{{tableStats.rawDataSize}}</td> + </tr> + <tr> + <td>Total Size</td> + <td>{{tableStats.totalSize}}</td> + </tr> + </tbody> + </table> + </div> + {{else}} + <div class="alert alert-danger"> + <p>Table statistics are not computed</p> + </div> + {{/if}} + +</div> + +<div class="row stats-section"> + <p><strong>COLUMNS STATISTICS</strong></p> + <table class="table table-bordered table-hover"> + <thead> + <tr> + <td width="50%">COLUMN NAME</td> + <td width="50%">STATISTICS</td> + </tr> + </thead> + <tbody> + {{#each columns as |column|}} + <tr> + <td>{{column.name}}</td> + <td> + {{#if column.hasStatistics}} + {{#if column.stats}} + <button class="btn btn-success btn-sm" {{action "toggleShowStats" column}}> + {{fa-icon "location-arrow"}} + {{#if column.showStats}} + Hide + {{else}} + Show + {{/if}} + </button> + {{else}} + <button class="btn btn-success btn-sm" disabled={{or column.isFetchingStats column.stats}} {{action "fetchStats" column}}> + {{#if column.isFetchingStats}} + {{fa-icon "cog" spin=true}} Fetching Stats + {{else}} + {{fa-icon "location-arrow"}} Show + {{/if}} + </button> + {{/if}} + + {{#if (and column.stats column.showStats)}} + <div class="col-stats-details"> + <table class="table table-bordered table-hover "> + <thead> + <tr> + <td width="50%">KEY</td> + <td width="50%">VALUE</td> + </tr> + </thead> + <tbody> + {{#each column.stats as |stat| }} + <tr> + <td>{{stat.label}}</td> + <td>{{stat.value}}</td> + </tr> + {{/each}} + </tbody> + </table> + </div> + {{/if}} + {{else}} + No statistics computed + {{/if}} + </td> + </tr> + {{/each}} + </tbody> + </table> +</div> + +{{#if showAnalyseModal}} + {{#modal-dialog + translucentOverlay=true + clickOutsideToClose=false + container-class="modal-dialog"}} + <div class="modal-content"> + <div class="modal-header"> + <p class="modal-title">{{fa-icon "location-arrow" size="lg"}} {{analyseTitle}}</p> + </div> + <div class="modal-body text-center"> + <div class="alert alert-danger"> + Analyse table statistics operation may take long time + </div> + <p><strong>{{analyseMessage}}</strong></p> + </div> + </div> + {{/modal-dialog}} +{{/if}} http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables.hbs b/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables.hbs index 1f98b97..c2d845b 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables.hbs +++ b/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables.hbs @@ -16,7 +16,7 @@ * limitations under the License. }} -<div class="dipayan123 clearfix"> +<div class="scroll-fix clearfix"> <div class="col-md-3"> <div class="row"> <div class="hv-dropdown tables-dropdown"> http://git-wip-us.apache.org/repos/asf/ambari/blob/22d4e181/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/table/stats.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/table/stats.hbs b/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/table/stats.hbs index 6671b8b..c522ac8 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/table/stats.hbs +++ b/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/table/stats.hbs @@ -14,4 +14,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -}} \ No newline at end of file +}} + +{{table-statistics table=table refresh="refreshTableInfo"}} \ No newline at end of file
