AMBARI-19999 : Hive view 2.0 upload table : supporting endlines in input file, supporting char datatype in column type, handing errors during upload (nitirajrathore)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/141e88dd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/141e88dd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/141e88dd Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 141e88dd4f3e2e48b2bd40d205e6c9716aec2669 Parents: 513b527 Author: Nitiraj Singh Rathore <[email protected]> Authored: Wed Feb 15 18:35:33 2017 +0530 Committer: Nitiraj Singh Rathore <[email protected]> Committed: Wed Feb 15 18:36:10 2017 +0530 ---------------------------------------------------------------------- .../resources/ui/app/components/radio-button.js | 1 - .../resources/ui/app/components/upload-table.js | 4 +- .../resources/ui/app/locales/en/translations.js | 14 +--- .../src/main/resources/ui/app/models/column.js | 7 +- .../databases/database/tables/upload-table.js | 79 +++++--------------- .../templates/components/csv-format-params.hbs | 18 +++-- .../templates/databases/database/tables/new.hbs | 2 +- 7 files changed, 42 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/141e88dd/contrib/views/hive20/src/main/resources/ui/app/components/radio-button.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/components/radio-button.js b/contrib/views/hive20/src/main/resources/ui/app/components/radio-button.js index 066168c..c2e7e0d 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/components/radio-button.js +++ b/contrib/views/hive20/src/main/resources/ui/app/components/radio-button.js @@ -28,7 +28,6 @@ export default Ember.Component.extend({ }.property('value', 'checked'), change: function() { - console.log("value changed : ", this.get('value')); this.set('checked', this.get('value')); }, http://git-wip-us.apache.org/repos/asf/ambari/blob/141e88dd/contrib/views/hive20/src/main/resources/ui/app/components/upload-table.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/components/upload-table.js b/contrib/views/hive20/src/main/resources/ui/app/components/upload-table.js index 29e9891..8df03e5 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/components/upload-table.js +++ b/contrib/views/hive20/src/main/resources/ui/app/components/upload-table.js @@ -23,6 +23,7 @@ export default Ember.Component.extend({ fileFormatInfo: Ember.Object.create({ csvParams: Ember.Object.create(), inputFileType: null, + containsEndlines: false, }), fileInfo: Ember.Object.create({ files: Ember.A(), @@ -33,9 +34,6 @@ export default Ember.Component.extend({ actions: { onFileChanged: function () { console.log("inside files changed"); - console.log("fileFormatInfo : ", this.get("fileFormatInfo")); - console.log("fileInfo : ", this.get("fileInfo")); - console.log("tableInfo : ", this.get("tableInfo")); this.send("preview"); }, preview: function () { http://git-wip-us.apache.org/repos/asf/ambari/blob/141e88dd/contrib/views/hive20/src/main/resources/ui/app/locales/en/translations.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/locales/en/translations.js b/contrib/views/hive20/src/main/resources/ui/app/locales/en/translations.js index b550dbe..50eba3a 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/locales/en/translations.js +++ b/contrib/views/hive20/src/main/resources/ui/app/locales/en/translations.js @@ -35,29 +35,23 @@ export default { 'escapeCharacterTooltip': "Escape character. Default is backslash (\).", 'quoteCharacterTooltip': 'Quote character. Default is double quote (").', 'quoteCharacterField': "Quote Character", + 'isFirstRowHeader': "Is first row header?", + 'fieldsTerminatedByTooltip': "Fields Terminated By character for Hive table.", + 'isFirstRowHeaderTooltip': "Check if the first row of CSV is a header.", + 'containsEndlines': "Contains endlines?", }, "uploadTable": { 'uploadProgress': "Upload Progress", 'uploading': "Uploading..", 'selectFromLocal': "Select from local", 'hdfsPath': "HDFS Path", - 'selectDatabase': "Select a Database", 'tableName': "Table name", 'tableNameErrorMessage': "Only alphanumeric and underscore characters are allowed in table name.", 'tableNameTooltip': "Enter valid (alphanumeric + underscore) table name.", - 'storedAs': "Stored as", - 'isFirstRowHeader': "Is first row header ?", - 'columnNameTooltip': "Enter valid (alphanumeric + underscore) column name.", 'columnNameErrorMessage': "Only alphanumeric and underscore characters are allowed in column names.", 'hdfsFieldTooltip': "Enter full HDFS path", 'hdfsFieldPlaceholder': "Enter full HDFS path", 'hdfsFieldErrorMessage': "Please enter complete path of hdfs file to upload.", - 'containsEndlines': "Contains endlines?", - 'fieldsTerminatedByField': "Fields Terminated By", - 'escapedByField': "Escape By", - 'escapedByTooltip': "Escaped By character for Hive table.", - 'fieldsTerminatedByTooltip': "Fields Terminated By character for Hive table.", - 'isFirstRowHeaderTooltip': "Check if the first row of CSV is a header.", 'showPreview': "Preview" } }, http://git-wip-us.apache.org/repos/asf/ambari/blob/141e88dd/contrib/views/hive20/src/main/resources/ui/app/models/column.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/models/column.js b/contrib/views/hive20/src/main/resources/ui/app/models/column.js index 73a9824..9480616 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/models/column.js +++ b/contrib/views/hive20/src/main/resources/ui/app/models/column.js @@ -111,10 +111,10 @@ let Column = Ember.Object.extend(Ember.Copyable,{ }, copy: function(){ - return Column.create({ + let col = Column.create({ name: this.get("name"), - type: this.get("type"), - precision: this.get("percision"), + type: datatypes.findBy("label", this.get("type.label")), + precision: this.get("precision"), scale: this.get("scale"), isPartitioned: this.get("isPartitioned"), isClustered: this.get("isClustered"), @@ -123,6 +123,7 @@ let Column = Ember.Object.extend(Ember.Copyable,{ errors: this.get("errors").copy(), editing: this.get("editing"), }); + return col; } }); http://git-wip-us.apache.org/repos/asf/ambari/blob/141e88dd/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js b/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js index 0e61905..a9bf9ea 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js +++ b/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js @@ -41,11 +41,6 @@ export default NewTable.extend({ getCharOptionByCharCode: function(charCode){ return Helpers.getAllTerminationCharacters().findBy("id", charCode + ""); }, - // onChangeSelectedFileType: function(){ - // if(this.get('selectedFileType') === this.get('fileTypes')[1] && this.get('containsEndlines') === true){ - // this.set('containsEndlines', false); - // } - // }.observes("selectedFileType", "containsEndlines"), getUploader(){ return this.get('store').adapterFor('upload-table'); }, @@ -102,7 +97,6 @@ export default NewTable.extend({ var self = this; var fetchJobPromise = this.get('jobService').getJob(jobId); fetchJobPromise.then(function (data) { - console.log("waitForJobStatus : data : ", data); var job = JSON.parse(JSON.stringify(data)); var status = job.status; if (status == constants.statuses.succeeded ) { @@ -262,27 +256,24 @@ export default NewTable.extend({ this.pushUploadProgressInfos(this.formatMessage('hive.messages.failedToCreateActualTable')); this.setError(error); }, + copyTableMeta: function(tableMeta){ + let colArray = Ember.copy(tableMeta.columns, true); + let tableMetaCopy = JSON.parse(JSON.stringify(tableMeta)); + tableMetaCopy.columns = colArray; + return tableMetaCopy; + }, createTempTable: function (tableData) { - let tableMeta = JSON.parse(JSON.stringify(tableData.get("tableMeta"))); - // manually copy the columns as they are missing members when copying - let columns = tableData.get("tableMeta").columns.map(function(col){ - return col.copy(); - }); - tableMeta.columns = columns; - - console.log("tableMeta : ", tableMeta); - - var self = this; console.log("createTempTable"); this.pushUploadProgressInfos(this.formatMessage('hive.messages.startingToCreateTemporaryTable')); + let tableMeta = this.copyTableMeta(tableData.get("tableMeta")); // deep copy or otherwise it does make separate var tempTableName = this.generateTempTableName(); tableMeta.name = tempTableName; var headers = tableMeta.columns.map(function(column){ if(tableData.fileFormatInfo.containsEndlines){ - column.type.label = "STRING"; - delete column.scale; - delete column.precision; + column.set("type", datatypes.findBy("label","STRING")); + column.set("scale"); + column.set("precision"); } return column; }); @@ -714,54 +705,21 @@ export default NewTable.extend({ this.set("showMoreOrLess", "Show Less"); } }, + validateInputs: function(tableData){ + let tableMeta = tableData.get("tableMeta"); + let containsEndlines = tableData.get("fileFormatInfo.containsEndlines"); + if(containsEndlines == true && tableMeta.settings && tableMeta.settings.fileFormat + && tableMeta.settings.fileFormat.type && tableMeta.settings.fileFormat.type === "TEXTFILE"){ + throw new Error(`Cannot support endlines in fields when the File Format is TEXTFILE. Please uncheck '${this.translate('hive.ui.csvFormatParams.containsEndlines')}'`); + } + }, - displayOption: "display:none", actions: { - toggleCSVFormat: function() { - console.log("inside toggleCSVFormat"); - this.toggleProperty('showCSVFormatInput'); - }, - hideInputParamModal : function(){ - Ember.$("#inputParamsModal").modal("hide"); - }, - showInputParamModal : function(){ - if(this.get('inputFileTypeCSV')){ - Ember.$("#inputParamsModal").modal("show"); - } - }, - hideRowFormatModal : function(){ - Ember.$("#rowFormatModal").modal("hide"); - }, - showRowFormatModal : function(){ - if(this.get('storedAsTextFile')) { - Ember.$("#rowFormatModal").modal("show"); - } - }, - toggleErrors: function () { - this.toggleProperty('showErrors'); - }, - // filesUploaded: function (files) { - // console.log("upload-table.js : uploaded new files : ", files); - // this.clearFields(); - // - // this.set('files', files); - // var name = files[0].name; - // var i = name.indexOf("."); - // var tableName = name.substr(0, i); - // this.set('tableName', tableName); - // var self = this; - // return this.generatePreview(sourceObject) - // }, preview: function (previewObject) { console.log("upload-table.js : uploaded new files : ", previewObject); this.clearFields(); this.set('previewObject', previewObject); - // var name = previewObject.get("fileInfo").get("files")[0].name; - // var i = name.indexOf("."); - // var tableName = name.substr(0, i); - // this.set('tableName', tableName); - // var self = this; return this.generatePreview(previewObject) }, previewFromHdfs: function () { @@ -770,6 +728,7 @@ export default NewTable.extend({ uploadTable: function (tableData) { console.log("tableData", tableData); try { + this.validateInputs(tableData); this.createTableAndUploadFile(tableData); } catch (e) { console.log("exception occured : ", e); http://git-wip-us.apache.org/repos/asf/ambari/blob/141e88dd/contrib/views/hive20/src/main/resources/ui/app/templates/components/csv-format-params.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/templates/components/csv-format-params.hbs b/contrib/views/hive20/src/main/resources/ui/app/templates/components/csv-format-params.hbs index c63f502..df80260 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/templates/components/csv-format-params.hbs +++ b/contrib/views/hive20/src/main/resources/ui/app/templates/components/csv-format-params.hbs @@ -113,16 +113,24 @@ <div class="row"> <div class="col-md-6 form-horizontal"> <div class="form-group"> - <label class="col-md-2 control-label">Is First Row Header</label> - <div class="col-md-4"> - <label> - {{input type="checkbox" checked=fileFormatInfo.csvParams.isFirstRowHeader}} - </label> + <label class="col-md-4 control-label">{{t 'hive.ui.csvFormatParams.isFirstRowHeader'}}</label> + <div class="col-md-8"> + {{input type="checkbox" checked=fileFormatInfo.csvParams.isFirstRowHeader}} </div> </div> </div> </div> {{/if}} + <div class="row"> + <div class="col-md-6 form-horizontal"> + <div class="form-group"> + <label class="col-md-4 control-label">{{t 'hive.ui.csvFormatParams.containsEndlines'}}</label> + <div class="col-md-8"> + {{input type="checkbox" checked=fileFormatInfo.containsEndlines}} + </div> + </div> + </div> + </div> </div> {{/if}} </div> http://git-wip-us.apache.org/repos/asf/ambari/blob/141e88dd/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/new.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/new.hbs b/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/new.hbs index 4f3b98a..288c00c 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/new.hbs +++ b/contrib/views/hive20/src/main/resources/ui/app/templates/databases/database/tables/new.hbs @@ -34,7 +34,7 @@ <div class="table-header row"> <p class="text-uppercase">table<strong> > create table</strong> <div class="pull-right"> - {{#link-to "databases.database.tables.upload-table" }}<p class="text-uppercase">{{fa-icon "upload"}} upload table</p>{{/link-to}} + {{#link-to "databases.database.tables.upload-table" }}<button class="btn btn-success"><p class="text-uppercase">{{fa-icon "upload"}} upload table</p></button>{{/link-to}} </div> </p> </div>
