Repository: ambari Updated Branches: refs/heads/trunk bb3b7525f -> c5aafc8fb
http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/main/resources/ui/hive-web/app/controllers/upload-table.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/controllers/upload-table.js b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/controllers/upload-table.js index 6fa6cfd..061a445 100644 --- a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/controllers/upload-table.js +++ b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/controllers/upload-table.js @@ -22,6 +22,44 @@ import constants from 'hive/utils/constants'; export default Ember.Controller.extend({ + DEFAULT_CSV_DELIMITER: ',', + DEFAULT_CSV_QUOTE: '"', + DEFAULT_CSV_ESCAPE: '\\', + NON_PRINTABLE_CHARS:[{"id":"0", "name":"NUL", "description":"(null)"}, + {"id":"1", "name":"SOH", "description":"(start of heading)"}, + {"id":"2", "name":"STX", "description":"(start of text)"}, + {"id":"3", "name":"ETX", "description":"(end of text)"}, + {"id":"4", "name":"EOT", "description":"(end of transmission)"}, + {"id":"5", "name":"ENQ", "description":"(enquiry)"}, + {"id":"6", "name":"ACK", "description":"(acknowledge)"}, + {"id":"7", "name":"BEL", "description":"(bell)"}, + {"id":"8", "name":"BS", "description":"(backspace)"}, + {"id":"9", "name":"TAB", "description":"(horizontal tab)"}, + {"id":"10", "name":"LF", "description":"(NL line feed - new line)"}, + {"id":"11", "name":"VT", "description":"(vertical tab)"}, + {"id":"12", "name":"FF", "description":"(NP form feed - new page)"}, + {"id":"13", "name":"CR", "description":"(carriage return)"}, + {"id":"14", "name":"SO", "description":"(shift out)"}, + {"id":"15", "name":"SI", "description":"(shift in)"}, + {"id":"16", "name":"DLE", "description":"(data link escape)"}, + {"id":"17", "name":"DC1", "description":"(device control 1)"}, + {"id":"18", "name":"DC2", "description":"(device control 2)"}, + {"id":"19", "name":"DC3", "description":"(device control 3)"}, + {"id":"20", "name":"DC4", "description":"(device control 4)"}, + {"id":"21", "name":"NAK", "description":"(negative ackowledge)"}, + {"id":"22", "name":"SYN", "description":"(synchronous idle)"}, + {"id":"23", "name":"ETB", "description":"(end of trans. block)"}, + {"id":"24", "name":"CAN", "description":"(cancel)"}, + {"id":"25", "name":"EM", "description":"(end of medium)"}, + {"id":"26", "name":"SUB", "description":"(substitute)"}, + {"id":"27", "name":"ESC", "description":"(escape)"}, + {"id":"28", "name":"FS", "description":"(file separator)"}, + {"id":"29", "name":"GS", "description":"(group separator)"}, + {"id":"30", "name":"RS", "description":"(record separator)"}, + {"id":"31", "name":"US", "description":"(unit separator)"}, + {"id":"32", "name":"Space", "description":""}, + {"id":"127", "name":"DEL", "description":""} + ], COLUMN_NAME_REGEX: "^[a-zA-Z]{1}[a-zA-Z0-9_]*$", TABLE_NAME_REGEX: "^[a-zA-Z]{1}[a-zA-Z0-9_]*$", HDFS_PATH_REGEX: "^[/]{1}.+", // unix path allows everything but here we have to mention full path so starts with / @@ -48,38 +86,22 @@ export default Ember.Controller.extend({ uploadProgressInfos : [], DEFAULT_DB_NAME : 'default', showPreview : false, - onChangeUploadSource : function(){ - this.clearFields(); - }.observes("uploadSource"), - setDefaultDB : function(){ - var self = this; - var defaultDatabase = this.get('databases').find( - function(item,index){ - if(item.id == self.DEFAULT_DB_NAME ) - return true; - } - ); - - console.log("setting the initial database to : " + defaultDatabase); - self.set("selectedDatabase",defaultDatabase); - }, - init: function() { - this.setDefaultDB(); - }, - uploadProgressInfo : Ember.computed("uploadProgressInfos.[]",function(){ - var info = ""; - for( var i = 0 ; i < this.get('uploadProgressInfos').length ; i++) - info += this.get('uploadProgressInfos').objectAt(i); - - return new Ember.Handlebars.SafeString(info); - }), + containsEndlines: false, inputFileTypes :[ {id : "CSV", name : "CSV"}, {id : "JSON", name : "JSON"}, {id : "XML", name : "XML"} ], - inputFileType : {id : "CSV", name : "CSV"}, + inputFileType: null, inputFileTypeCSV : Ember.computed.equal('inputFileType.id',"CSV"), + storedAsTextFile : Ember.computed.equal("selectedFileType","TEXTFILE"), + storedAsNotTextFile : Ember.computed.not("storedAsTextFile"), + csvDelimiter: null, + csvQuote : null, + csvEscape : null, + asciiList:[], + fieldsTerminatedBy: null, + escapedBy: null, fileTypes:[ "SEQUENCEFILE", "TEXTFILE" , @@ -87,10 +109,13 @@ export default Ember.Controller.extend({ "ORC" , "PARQUET" , "AVRO" - //, - //"INPUTFORMAT" -- not supported as of now. ], - selectedFileType: "ORC", + selectedFileType: null, + onChangeSelectedFileType: function(){ + if(this.get('selectedFileType') === this.get('fileTypes')[1] && this.get('containsEndlines') === true){ + this.set('containsEndlines', false); + } + }.observes("selectedFileType", "containsEndlines"), dataTypes: [ "TINYINT", // "SMALLINT", // @@ -107,9 +132,61 @@ export default Ember.Controller.extend({ "VARCHAR", // -- (Note: Available in Hive 0.12.0 and later) "CHAR" // -- (Note: Available in Hive 0.13.0 and later) ], + setDefaultDB : function(){ + var self = this; + var defaultDatabase = this.get('databases').find( + function(item,index){ + if(item.id == self.DEFAULT_DB_NAME ) + return true; + } + ); + + console.log("setting the initial database to : " + defaultDatabase); + self.set("selectedDatabase",defaultDatabase); + }, + init: function () { + this.setDefaultDB(); + this.fillAsciiList(); + this.set("selectedFileType", this.get("fileTypes")[3]); + this.set("inputFileType", this.get("inputFileTypes")[0]); + }, + onChangeUploadSource : function(){ + this.clearFields(); + }.observes("uploadSource"), + fillAsciiList: function(){ + var list = this.get('asciiList'); + list.push({"id": -1, "name": ""}); + var nonPrintable = this.get('NON_PRINTABLE_CHARS'); + for( var i = 0 ; i <= 127 ; i++ ){ + var charInfo = nonPrintable.find(function(item){ + return item.id == i; + }); + if(!charInfo){ + charInfo = {"id": i, "name": String.fromCodePoint(i), "description":"" }; + } + var option = {"id": i, "name": charInfo.id + " " + charInfo.name + charInfo.description}; + list.push(option); + if(i === 44){ + this.set("csvDelimiter", option); + } + else if(i === 34){ + this.set("csvQuote", option); + } + else if(i === 92){ + this.set("csvEscape", option); + } + } + }, + uploadProgressInfo : Ember.computed("uploadProgressInfos.[]",function(){ + var info = ""; + for( var i = 0 ; i < this.get('uploadProgressInfos').length ; i++) + info += this.get('uploadProgressInfos').objectAt(i); + + return new Ember.Handlebars.SafeString(info); + }), _setHeaderElements : function(header,valueArray){ header.forEach(function (item, index) { - Ember.set(item, 'name', valueArray.objectAt(index)); + Ember.set(item, 'name', valueArray[index]); }, this); }, isFirstRowHeaderDidChange: function () { @@ -158,8 +235,9 @@ export default Ember.Controller.extend({ this.set("hdfsPath"); this.set("header"); this.set("rows"); + this.set("escapedBy"); + this.set("fieldsTerminatedBy"); this.set("error"); - this.set('isFirstRowHeader',false); this.set('files'); this.set("firstRow"); this.set("selectedDatabase",null); @@ -203,8 +281,10 @@ export default Ember.Controller.extend({ console.log("rejecting waitForJobStatus with : " + status); reject(new Error(job.statusMessage)); } else { - console.log("retrying waitForJobStatus : ", jobId); - self.waitForJobStatus(jobId, resolve, reject); + Ember.run.later(function(){ + console.log("retrying waitForJobStatus : ", jobId); + self.waitForJobStatus(jobId, resolve, reject); + },1000); } }, function (error) { console.log("rejecting waitForJobStatus with : " + error); @@ -215,20 +295,56 @@ export default Ember.Controller.extend({ uploadForPreview: function (files) { console.log("uploaderForPreview called."); var self = this; + var csvParams = this.getCSVParams(); + return this.get('uploader').uploadFiles('preview', files, { "isFirstRowHeader": self.get("isFirstRowHeader"), - "inputFileType": self.get("inputFileType").id + "inputFileType": self.get("inputFileType").id, + "csvDelimiter": csvParams.csvDelimiter, + "csvEscape": csvParams.csvEscape, + "csvQuote": csvParams.csvQuote }); }, + getAsciiChar : function(key){ + if(!key){ + return null; + } + + var value = this.get(key); + if(value && value.id != -1) { + return String.fromCharCode(value.id); + }else{ + return null; + } + }, + getCSVParams : function(){ + var csvd = this.getAsciiChar('csvDelimiter'); + if(!csvd && csvd != 0) csvd = this.get('DEFAULT_CSV_DELIMITER'); + + var csvq = this.getAsciiChar('csvQuote'); + if(!csvq && csvq != 0) csvq = this.get('DEFAULT_CSV_QUOTE'); + + var csve = this.getAsciiChar('csvEscape'); + if(!csve && csve != 0) csve = this.get('DEFAULT_CSV_ESCAPE'); + + return {"csvDelimiter": csvd, "csvQuote" : csvq, "csvEscape": csve}; + }, + uploadForPreviewFromHDFS: function () { console.log("uploadForPreviewFromHDFS called."); + var self = this; var hdfsPath = this.get("hdfsPath"); this.validateHDFSPath(hdfsPath); + var csvParams = this.getCSVParams(); + return this.get('uploader').previewFromHDFS({ "isFirstRowHeader": this.get("isFirstRowHeader"), "inputFileType": this.get("inputFileType").id, - "hdfsPath": hdfsPath + "hdfsPath": hdfsPath, + "csvDelimiter": csvParams.csvDelimiter, + "csvEscape": csvParams.csvEscape , + "csvQuote": csvParams.csvQuote }); }, @@ -273,16 +389,25 @@ export default Ember.Controller.extend({ console.log('inside previewTable'); var self = this; var defaultColumnNames = data.header.map(function(item,index){ - return self.COLUMN_NAME_PREFIX + index; + return self.COLUMN_NAME_PREFIX + (index + 1); }); this.set("defaultColumnNames",defaultColumnNames); this.set("header", data.header); - this.set("firstRow", data.rows[0].row); this.set('isFirstRowHeader', data.isFirstRowHeader); this.set('tableName', data.tableName); + var firstRow = null; if (data.isFirstRowHeader == true) { - data.rows = data.rows.slice(1); + firstRow = data.header.map(function(columnDesc){ + return columnDesc.name; + }); + }else { + if(data.rows.length > 0){ + firstRow = data.rows[0].row; + }else{ + firstRow = []; + } } + this.set("firstRow", firstRow); this.set("rows", data.rows); }, @@ -302,6 +427,7 @@ export default Ember.Controller.extend({ createActualTable: function () { console.log("createActualTable"); + var self = this; this.pushUploadProgressInfos(this.formatMessage('hive.messages.startingToCreateActualTable')); var headers = this.get('header'); var selectedDatabase = this.get('selectedDatabase'); @@ -309,7 +435,7 @@ export default Ember.Controller.extend({ throw new Error(this.translate('hive.errors.emptyDatabase', {database : this.translate("hive.words.database")})); } - this.set('databaseName', this.get('selectedDatabase').get('name')); + this.set('databaseName', this.get('selectedDatabase.id')); var databaseName = this.get('databaseName'); var tableName = this.get('tableName'); var isFirstRowHeader = this.get('isFirstRowHeader'); @@ -317,16 +443,21 @@ export default Ember.Controller.extend({ this.validateInput(headers,tableName,databaseName,isFirstRowHeader); this.showUploadModal(); - + var rowFormat = this.getRowFormat(); return this.get('uploader').createTable({ "isFirstRowHeader": isFirstRowHeader, "header": headers, "tableName": tableName, "databaseName": databaseName, - "fileType":filetype + "hiveFileType":filetype, + "rowFormat": { "fieldsTerminatedBy" : rowFormat.fieldsTerminatedBy, "escapedBy" : rowFormat.escapedBy} }); }, - + getRowFormat : function(){ + var fieldsTerminatedBy = this.getAsciiChar('fieldsTerminatedBy'); + var escapedBy = this.getAsciiChar('escapedBy'); + return {"fieldsTerminatedBy": fieldsTerminatedBy, "escapedBy" : escapedBy}; + }, waitForCreateActualTable: function (jobId) { console.log("waitForCreateActualTable"); this.popUploadProgressInfos(); @@ -338,31 +469,39 @@ export default Ember.Controller.extend({ return p; }, - onCreateActualTableSuccess: function () { console.log("onCreateTableSuccess"); this.popUploadProgressInfos(); this.pushUploadProgressInfos(this.formatMessage('hive.messages.successfullyCreatedActualTable')); }, - onCreateActualTableFailure: function (error) { console.log("onCreateActualTableFailure"); this.popUploadProgressInfos(); this.pushUploadProgressInfos(this.formatMessage('hive.messages.failedToCreateActualTable')); this.setError(error); }, - createTempTable: function () { + var self = this; console.log("createTempTable"); this.pushUploadProgressInfos(this.formatMessage('hive.messages.startingToCreateTemporaryTable')); var tempTableName = this.generateTempTableName(); this.set('tempTableName', tempTableName); + + var headers = this.get("header"); + if(this.get("containsEndlines")){ + headers = this.get("header").map(function(item){ + var header = JSON.parse(JSON.stringify(item)); + header.type = "STRING"; + return header; + }); + } return this.get('uploader').createTable({ "isFirstRowHeader": this.get("isFirstRowHeader"), - "header": this.get("header"), + "header": headers, "tableName": tempTableName, "databaseName": this.get('databaseName'), - "fileType": "TEXTFILE" + "hiveFileType":"TEXTFILE", + "rowFormat": { "fieldsTerminatedBy" : parseInt('1', 10), "escapedBy" : null} }); }, @@ -398,9 +537,9 @@ export default Ember.Controller.extend({ var self = this; this.pushUploadProgressInfos(this.formatMessage('hive.messages.deletingTable',{table:tableLabel})); - return this.deleteTable(databaseName, tableName).then(function (data) { + return this.deleteTable(databaseName, tableName).then(function (job) { return new Ember.RSVP.Promise(function (resolve, reject) { - self.waitForJobStatus(data.jobId, resolve, reject); + self.waitForJobStatus(job.id, resolve, reject); }); }).then(function () { self.popUploadProgressInfos(); @@ -500,7 +639,9 @@ export default Ember.Controller.extend({ "fromDatabase": this.get("databaseName"), "fromTable": this.get("tempTableName"), "toDatabase": this.get("databaseName"), - "toTable": this.get("tableName") + "toTable": this.get("tableName"), + "header": this.get("header"), + "unhexInsert": this.get("containsEndlines") }); }, @@ -543,7 +684,6 @@ export default Ember.Controller.extend({ this.get("tempTableName") ); }, - waitForDeleteTempTable: function (jobId) { console.log("waitForDeleteTempTable"); this.popUploadProgressInfos(); @@ -555,20 +695,23 @@ export default Ember.Controller.extend({ return p; }, - onDeleteTempTableSuccess: function () { console.log("onDeleteTempTableSuccess"); this.popUploadProgressInfos(); this.pushUploadProgressInfos(this.formatMessage('hive.messages.successfullyDeletedTemporaryTable')); this.onUploadSuccessfull(); }, - onDeleteTempTableFailure: function (error) { console.log("onDeleteTempTableFailure"); this.setError(error); this.setError(this.formatMessage('hive.messages.manuallyDeleteTable',{databaseName:this.get('databaseName'), tableName: this.get("tempTableName")})); }, - + validateHDFSPath: function (hdfsPath) { + if (null == hdfsPath || hdfsPath == "") throw new Error(this.translate('hive.errors.emptyHdfsPath')); + var hdfsRegex = new RegExp(this.get("HDFS_PATH_REGEX"), "g"); + var mArr = hdfsPath.match(hdfsRegex); + if (mArr == null || mArr.length != 1) throw new Error(this.translate('hive.errors.illegalHdfPath', {"hdfsPath": hdfsPath} )); + }, createTableAndUploadFile: function () { var self = this; self.setError(); @@ -664,14 +807,15 @@ export default Ember.Controller.extend({ self.onDeleteTempTableFailure(error); } throw error; - }).catch(function(error){ + }) + .catch(function(error){ console.log("inside catch : ", error); - }).finally(function(){ + }) + .finally(function(){ console.log("finally hide the modal always"); self.hideUploadModal(); }); }, - validateInput: function (headers,tableName,databaseName,isFirstRowHeader) { // throw exception if invalid. if(!headers || headers.length == 0) throw new Error(this.translate('hive.errors.emptyHeaders')); @@ -695,41 +839,52 @@ export default Ember.Controller.extend({ throw new Error(this.translate('hive.errors.emptyIsFirstRow', {isFirstRowHeaderField:this.translate('hive.ui.isFirstRowHeader')})); } }, - setError: function (error) { if(error){ - console.log("upload table error : ", error); + console.log(" error : ", error); this.set('error', JSON.stringify(error)); this.get('notifyService').error(error); }else{ this.set("error"); } }, - previewError: function (error) { this.setError(error); }, - uploadTableFromHdfs : function(){ console.log("uploadTableFromHdfs called."); if(!(this.get("inputFileTypeCSV") == true && this.get("isFirstRowHeader") == false) ){ this.pushUploadProgressInfos(this.formatMessage('uploadingFromHdfs')); } - return this.get('uploader').uploadFromHDFS({ - "isFirstRowHeader": this.get("isFirstRowHeader"), - "databaseName" : this.get('databaseName'), - "tableName" : this.get("tempTableName"), - "inputFileType" : this.get("inputFileType").id, - "hdfsPath" : this.get("hdfsPath") - }); + var csvParams = this.getCSVParams(); + + return this.get('uploader').uploadFromHDFS({ + "isFirstRowHeader": this.get("isFirstRowHeader"), + "databaseName": this.get('databaseName'), + "tableName": this.get("tempTableName"), + "inputFileType": this.get("inputFileType").id, + "hdfsPath": this.get("hdfsPath"), + "header": this.get("header"), + "containsEndlines": this.get("containsEndlines"), + "csvDelimiter": csvParams.csvDelimiter, + "csvEscape": csvParams.csvEscape, + "csvQuote": csvParams.csvQuote + }); }, uploadTable: function () { this.printValues(); + var csvParams = this.getCSVParams(); + return this.get('uploader').uploadFiles('upload', this.get('files'), { "isFirstRowHeader": this.get("isFirstRowHeader"), "databaseName" : this.get('databaseName'), "tableName" : this.get("tempTableName"), - "inputFileType" : this.get("inputFileType").id + "inputFileType" : this.get("inputFileType").id, + "header": JSON.stringify(this.get("header")), + "containsEndlines": this.get("containsEndlines"), + "csvDelimiter": csvParams.csvDelimiter, + "csvEscape": csvParams.csvEscape , + "csvQuote": csvParams.csvQuote }); }, @@ -755,6 +910,22 @@ export default Ember.Controller.extend({ }, displayOption: "display:none", actions: { + hideInputParamModal : function(){ + Ember.$("#inputParamsModal").modal("hide"); + }, + showInputParamModal : function(){ + if(this.get('inputFileTypeCSV')){ + Ember.$("#inputParamsModal").modal("show"); + } + }, + hideRowFormatModal : function(){ + Ember.$("#rowFormatModal").modal("hide"); + }, + showRowFormatModal : function(){ + if(this.get('storedAsTextFile')) { + Ember.$("#rowFormatModal").modal("show"); + } + }, toggleErrors: function () { this.toggleProperty('showErrors'); }, http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/main/resources/ui/hive-web/app/initializers/i18n.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/initializers/i18n.js b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/initializers/i18n.js index a00f0b4..f7f7706 100644 --- a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/initializers/i18n.js +++ b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/initializers/i18n.js @@ -248,11 +248,13 @@ TRANSLATIONS = { 'no.query': "No query to process.", 'emptyDatabase' : "Please select {{ database }}.", 'emptyTableName' : "Please enter {{ tableNameField }}.", - 'illegalTableName':"Illegal {{ tableNameField }} : '{{ tableName }}'", + 'illegalTableName': "Illegal {{ tableNameField }} : '{{ tableName }}'", 'emptyIsFirstRow' : "{{isFirstRowHeaderField}} cannot be null.", - 'emptyHeaders':"Headers (containing column names) cannot be null.", - 'emptyColumnName':"Column name cannot be null.", - 'illegalColumnName':"Illegal column name : '{{columnName}}' in column number {{index}}", + 'emptyHeaders': "Headers (containing column names) cannot be null.", + 'emptyColumnName': "Column name cannot be null.", + 'illegalColumnName': "Illegal column name : '{{columnName}}' in column number {{index}}", + 'emptyHdfsPath': "HdfsPath Name cannot be null or empty.", + 'illegalHdfPath': "Illegal hdfs path : {{hdfsPath}}" }, messages : { 'generatingPreview':"Generating Preview.", @@ -286,7 +288,7 @@ TRANSLATIONS = { words :{ temporary : "Temporary", actual : "Actual", - database : "Database", + database : "Database" }, ui : { 'uploadProgress' : "Upload Progress", @@ -306,8 +308,19 @@ TRANSLATIONS = { 'columnNameErrorMessage':"Only alphanumeric and underscore characters are allowed in column names.", 'hdfsFieldTooltip':"Enter full HDFS path", 'hdfsFieldPlaceholder':"Enter full HDFS path", - 'hdfsFieldErrorMessage':"Please enter complete path of hdfs file to upload." - + 'hdfsFieldErrorMessage':"Please enter complete path of hdfs file to upload.", + 'containsEndlines': "Contains endlines?", + 'columnDelimiterTooltip': "Delimiter for the column values. Default is comman (,).", + 'escapeCharacterTooltip': "Escape character. Default is backslash (\).", + 'quoteCharacterTooltip': 'Quote character. Default is double quote (").', + 'quoteCharacterField': "Quote Character", + 'escapeCharacterField': "Escape Character", + 'columnDelimterField': "Field Delimiter", + 'fieldsTerminatedByField': "Fields Terminated By", + 'escapedByField': "Escape By", + 'escapedByTooltip': "Escaped By character for Hive table.", + 'fieldsTerminatedByTooltip': "Fields Terminated By character for Hive table.", + 'isFirstRowHeaderTooltip': "Check if the first row of CSV is a header." } }, http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/main/resources/ui/hive-web/app/styles/app.scss ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/styles/app.scss b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/styles/app.scss index f0c9a4c..7598b0e 100644 --- a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/styles/app.scss +++ b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/styles/app.scss @@ -703,3 +703,14 @@ pre.explainprint{ line-height: 0.5; padding: 12px 15px; } +.settings-gear { + padding:5px; + cursor: pointer; +} + +.settings-gear-disabled { + @extend .settings-gear; + + color:#ddd; + cursor: default; +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs index 5e3a519..d84dc4d 100644 --- a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs +++ b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs @@ -44,8 +44,105 @@ </div> </div> +<!-- CSV Input Modal --> +<div class="modal fade" id="inputParamsModal" role="dialog"> + <div class="modal-dialog"> + <div class="modal-content"> + <div class="modal-header"> + <button type="button" class="close" data-dismiss="modal">×</button> + </div> + <div class="modal-body"> + <table class="table data-upload-form"> + <tr> + <td><label>{{t 'hive.ui.columnDelimterField'}}:</label></td> + <td class="data-upload-form-field" title={{t 'hive.ui.columnDelimiterTooltip'}}> + {{typeahead-widget + content=asciiList + optionValuePath="id" + optionLabelPath="name" + selection=csvDelimiter + }} + </td> + </tr> + <tr> + <td><label>{{t 'hive.ui.escapeCharacterField'}}:</label></td> + <td class="data-upload-form-field" title={{t 'hive.ui.escapeCharacterTooltip'}}> + {{typeahead-widget + content=asciiList + optionValuePath="id" + optionLabelPath="name" + selection=csvEscape + }} + </td> + </tr> + <tr> + <td><label>{{t 'hive.ui.quoteCharacterField'}}:</label></td> + <td class="data-upload-form-field" title={{t 'hive.ui.quoteCharacterTooltip'}}> + {{typeahead-widget + content=asciiList + optionValuePath="id" + optionLabelPath="name" + selection=csvQuote + }} + </td> + </tr> + <tr> + <td><label>{{t "hive.ui.isFirstRowHeader"}}</label></td> + <td class="data-upload-form-field" title={{t 'hive.ui.isFirstRowHeaderTooltip'}}> + {{input id="isFirstRowHeader" type="checkbox" checked=isFirstRowHeader }} + </td> + </tr> + </table> + </div> + <div class="modal-footer"> + <button type="submit" class="btn btn-default btn-default pull-right" data-dismiss="modal"> Close</button> + </div> + </div> + </div> +</div> + +<!-- Row Format Modal --> +<div class="modal fade" id="rowFormatModal" role="dialog"> + <div class="modal-dialog"> + <div class="modal-content"> + <div class="modal-header"> + <button type="button" class="close" data-dismiss="modal">×</button> + </div> + <div class="modal-body"> + <table class="table data-upload-form"> + <tr> + <td><label>{{t 'hive.ui.fieldsTerminatedByField'}}:</label></td> + <td class="data-upload-form-field" title={{t 'hive.ui.fieldsTerminatedByTooltip'}}> + {{typeahead-widget + content=asciiList + optionValuePath="id" + optionLabelPath="name" + selection=fieldsTerminatedBy + }} + </td> + </tr> + <tr> + <td><label>{{t 'hive.ui.escapedByField'}}:</label></td> + <td class="data-upload-form-field" title={{t 'hive.ui.escapedByTooltip'}}> + {{typeahead-widget + content=asciiList + optionValuePath="id" + optionLabelPath="name" + selection=escapedBy + }} + </td> + </tr> + </table> + </div> + <div class="modal-footer"> + <button type="submit" class="btn btn-default btn-default pull-right" data-dismiss="modal"> Close</button> + </div> + </div> + </div> +</div> + <div class="pull-right"> - <i class="query-menu-tab fa queries-icon fa-envelope" {{ action 'toggleErrors'}}></i> + <i class="query-menu-tab fa queries-icon fa-envelope" {{ action 'toggleErrors'}} ></i> </div> <div {{bind-attr class="showErrors:hide-data:show-data"}}> <div> @@ -60,15 +157,20 @@ <tr> <td class="data-upload-form-label"><label>{{t "hive.ui.fileType"}}</label></td> <td class="data-upload-form-field"> - {{typeahead-widget - content=inputFileTypes - optionValuePath="id" - optionLabelPath="name" - selection=inputFileType - placeholder=(t "hive.ui.uploadFromHdfs") - }} + <div class="col-md-11" style="padding:0"> + {{typeahead-widget + content=inputFileTypes + optionValuePath="id" + optionLabelPath="name" + selection=inputFileType + placeholder="Select File Type"}} + </div> + <div class="col-md-1" style="padding: 0"> + <span {{bind-attr class=":queries-icon :fa :fa-gear inputFileTypeCSV:settings-gear:settings-gear-disabled"}} + title="Settings" {{action "showInputParamModal"}}> + </span> + </div> </td> - {{#if isLocalUpload }} <td class="data-upload-form-label"><label>{{t "hive.ui.selectFromLocal"}}</label></td> <td class="data-upload-form-field">{{file-upload filesUploaded="filesUploaded" uploadFiles=files}}</td> @@ -85,43 +187,48 @@ {{/if}} </tr> {{#if showPreview}} - <tr> - <td class="data-upload-form-label"><label>{{t "hive.words.database"}}</label></td> - <td class="data-upload-form-field"> - {{typeahead-widget - content=databases - optionValuePath="id" - optionLabelPath="name" - selection=selectedDatabase - placeholder=(t "hive.ui.selectDatabase") - }} - </td> + <tr> + <td class="data-upload-form-label"><label>{{t "hive.words.database"}}</label></td> + <td class="data-upload-form-field"> + {{typeahead-widget + content=databases + optionValuePath="id" + optionLabelPath="name" + selection=selectedDatabase + placeholder=(t "hive.ui.selectDatabase") + }} + </td> - <td class="data-upload-form-label"><label>{{t "hive.ui.tableName"}}</label></td> - <td - class="data-upload-form-field"> - {{#validated-text-field inputValue=tableName allowEmpty=false - tooltip=(t "hive.ui.tableNameTooltip") - invalidClass='form-control red-border' validClass='form-control' regex=TABLE_NAME_REGEX - errorMessage=(t "hive.ui.tableNameErrorMessage") }} - {{/validated-text-field}} - </td> - </tr> - <tr> - <td class="data-upload-form-label"><label>{{t "hive.ui.storedAs"}}</label></td> + <td class="data-upload-form-label"><label>{{t "hive.ui.tableName"}}</label></td> + <td class="data-upload-form-field"> + {{#validated-text-field inputValue=tableName allowEmpty=false + tooltip=(t "hive.ui.tableNameTooltip") + invalidClass='form-control red-border' validClass='form-control' regex=TABLE_NAME_REGEX + errorMessage=(t "hive.ui.tableNameErrorMessage") }} + {{/validated-text-field}} + </td> + </tr> + <tr> + <td class="data-upload-form-label"><label>{{t "hive.ui.storedAs"}}</label></td> + <td class="data-upload-form-field"> + <div class="col-md-11" style="padding: 0"> + {{typeahead-widget + content=fileTypes + selection=selectedFileType}} + </div> + <div class="col-md-1" style="padding: 0"> + <span {{bind-attr class=":queries-icon :fa :fa-gear storedAsTextFile:settings-gear:settings-gear-disabled"}} + title="Settings" {{action "showRowFormatModal"}}> + </span> + </div> + </td> + {{#if storedAsNotTextFile}} + <td class="data-upload-form-label"><label>{{t "hive.ui.containsEndlines"}}</label></td> <td class="data-upload-form-field"> - {{typeahead-widget - content=fileTypes - selection=selectedFileType}} + {{input type="checkbox" checked=containsEndlines }} </td> - {{#if inputFileTypeCSV }} - <td class="data-upload-form-label"><label>{{t "hive.ui.isFirstRowHeader"}}</label></td> - <td class="data-upload-form-field"> - {{input id="isFirstRowHeader" type="checkbox" checked=isFirstRowHeader }} - </td> - - {{/if}} - </tr> + {{/if}} + </tr> {{/if}} </table> http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java index fab1328..fa6fba3 100644 --- a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java @@ -19,7 +19,8 @@ package org.apache.ambari.view.hive2.resources.upload; import org.apache.ambari.view.hive2.client.Row; -import org.apache.ambari.view.hive2.resources.uploads.parsers.csv.CSVParser; +import org.apache.ambari.view.hive2.resources.uploads.parsers.ParseOptions; +import org.apache.ambari.view.hive2.resources.uploads.parsers.csv.commonscsv.CSVParser; import org.junit.Assert; import org.junit.Test; @@ -39,7 +40,7 @@ public class CSVParserTest { try( StringReader sr = new StringReader(csv); - CSVParser jp = new CSVParser(sr, null); + CSVParser jp = new CSVParser(sr, new ParseOptions()); ) { Assert.assertEquals("There should not be any rows.",false, jp.iterator().hasNext()); } @@ -55,7 +56,7 @@ public class CSVParserTest { try( StringReader sr = new StringReader(csv); - CSVParser jp = new CSVParser(sr, null); + CSVParser jp = new CSVParser(sr, new ParseOptions()); ) { Iterator<Row> iterator = jp.iterator(); @@ -70,7 +71,7 @@ public class CSVParserTest { try( StringReader sr = new StringReader(csv); - CSVParser jp = new CSVParser(sr, null); + CSVParser jp = new CSVParser(sr, new ParseOptions()); ) { Iterator<Row> iterator = jp.iterator(); @@ -91,7 +92,7 @@ public class CSVParserTest { try( StringReader sr = new StringReader(csv); - CSVParser jp = new CSVParser(sr, null); + CSVParser jp = new CSVParser(sr, new ParseOptions()); ) { Iterator<Row> iterator = jp.iterator(); @@ -106,4 +107,169 @@ public class CSVParserTest { Assert.assertEquals("Failed to detect end of rows 2nd time!", false, iterator.hasNext()); } } + + + @Test + public void testQuotedEndline() throws Exception { + + String csv = "\"row1-\ncol1\",1,1.1\n\"row2-\\\ncol1\",2,2.2\n"; + ParseOptions po = new ParseOptions(); + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"row1-\ncol1", "1", "1.1"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + + Row row2 = new Row(new Object[]{"row2-\\\ncol1", "2", "2.2"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row2, iterator.next()); + + } + } + + @Test + public void testQuotedDoubleQuote() throws Exception { + + String csv = "\"aaa\",\"b\"\"bb\",\"ccc\""; + ParseOptions po = new ParseOptions(); + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"aaa", "b\"bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + @Test + public void testSpecialEscape() throws Exception { + + String csv = "\"aaa\",\"b$\"bb\",\"ccc\""; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'$'); + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"aaa", "b\"bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + @Test + public void testSpecialEscapedEscape() throws Exception { + + String csv = "aaa,b$$bb,ccc"; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'$'); + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"aaa", "b$bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + @Test + public void test001Escape() throws Exception { + + String csv = "aaa,b\001\"bb,ccc"; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'\001'); + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + Row row = new Row(new Object[]{"aaa", "b\"bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); } + } + + @Test + public void testSpecialQuote() throws Exception { + + String csv = "\001aaa\001,\001b\001\001bb\001,\001ccc\001"; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_QUOTE,'\001'); + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + Row row = new Row(new Object[]{"aaa", "b\001bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + @Test + public void testSpaceAsDelimiterAndQuoted() throws Exception { + + String csv = "aaa \"b bb\" ccc\naaa2 bbb2 \"c cc2\""; + ParseOptions po = new ParseOptions(); +// po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'\001'); + po.setOption(ParseOptions.OPTIONS_CSV_DELIMITER,' '); + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + Row row = new Row(new Object[]{"aaa", "b bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + + Row row2 = new Row(new Object[]{"aaa2", "bbb2", "c cc2"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row2, iterator.next()); + } + } + + @Test + public void testFailedDelimiterEscaped() throws Exception { + + String csv = "aaa,b\\,bb,ccc"; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'\\'); + po.setOption(ParseOptions.OPTIONS_CSV_DELIMITER,','); + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + Row row = new Row(new Object[]{"aaa", "b,bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java index 6b57228..a97c7ea 100644 --- a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java @@ -184,8 +184,8 @@ public class DataParserCSVTest { Assert.assertNotNull(pd.getHeader()); Assert.assertEquals(1, pd.getPreviewRows().size()); Assert.assertEquals(2, pd.getHeader().size()); - ColumnDescription[] cd = {new ColumnDescriptionImpl("Column1", ColumnDescriptionShort.DataTypes.INT.toString(), 0), - new ColumnDescriptionImpl("Column2", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1)}; + ColumnDescription[] cd = {new ColumnDescriptionImpl("column1", ColumnDescriptionShort.DataTypes.INT.toString(), 0), + new ColumnDescriptionImpl("column2", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1)}; Object cols1[] = new Object[2]; cols1[0] = "1"; http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java index 1ea7ec2..6800967 100644 --- a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java @@ -246,8 +246,8 @@ public class DataParserJSONTest { Assert.assertNotNull(pd.getHeader()); Assert.assertEquals(1, pd.getPreviewRows().size()); Assert.assertEquals(2, pd.getHeader().size()); - ColumnDescription[] cd = {new ColumnDescriptionImpl("Column1", ColumnDescriptionShort.DataTypes.CHAR.toString(), 0), - new ColumnDescriptionImpl("Column2", ColumnDescriptionShort.DataTypes.STRING.toString(), 1)}; + ColumnDescription[] cd = {new ColumnDescriptionImpl("column1", ColumnDescriptionShort.DataTypes.CHAR.toString(), 0), + new ColumnDescriptionImpl("column2", ColumnDescriptionShort.DataTypes.STRING.toString(), 1)}; Object cols1[] = new Object[2]; cols1[0] = "d"; http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java index 2faf4e9..e2228bb 100644 --- a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java @@ -278,8 +278,8 @@ public class DataParserXMLTest { Assert.assertNotNull(pd.getHeader()); Assert.assertEquals(1, pd.getPreviewRows().size()); Assert.assertEquals(2, pd.getHeader().size()); - ColumnDescription[] cd = {new ColumnDescriptionImpl("Column1", ColumnDescriptionShort.DataTypes.STRING.toString(), 0), - new ColumnDescriptionImpl("Column2", ColumnDescriptionShort.DataTypes.INT.toString(), 1)}; + ColumnDescription[] cd = {new ColumnDescriptionImpl("column1", ColumnDescriptionShort.DataTypes.STRING.toString(), 0), + new ColumnDescriptionImpl("column2", ColumnDescriptionShort.DataTypes.INT.toString(), 1)}; Object cols1[] = new Object[2]; cols1[0] = "row1-col1-Value"; http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/OpenCSVParserTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/OpenCSVParserTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/OpenCSVParserTest.java new file mode 100644 index 0000000..5aaa33d --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/OpenCSVParserTest.java @@ -0,0 +1,313 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.parsers.ParseOptions; +import org.apache.ambari.view.hive2.resources.uploads.parsers.csv.opencsv.OpenCSVParser; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; + +public class OpenCSVParserTest { + + /** + * no exception in creating csvParser with emtpy stream + * @throws IOException + */ + @Test + public void testEmptyStream() throws Exception { + String csv = ""; + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, new ParseOptions()); + ) { + Assert.assertEquals("There should not be any rows.",false, jp.iterator().hasNext()); + } + } + + /** + * in case of csv an empty line is still considered as row + * @throws IOException + */ + @Test + public void testEmptyRow() throws Exception { + String csv = " "; + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, new ParseOptions()); + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator should be Empty", true, iterator.hasNext()); + Assert.assertArrayEquals("Row should not be empty",new Object[]{" "},iterator.next().getRow()); + } + } + + @Test + public void testParse1Row() throws Exception { + String csv = "value1,c,10,10.1"; + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, new ParseOptions()); + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator Empty!", true, iterator.hasNext()); + Row row = iterator.next(); + Row expected = new Row(new Object[]{"value1", "c", "10", "10.1"}); + Assert.assertEquals("Row not equal!", expected, row); + + Assert.assertEquals("Should report no more rows!", false, iterator.hasNext()); + } + } + + @Test + public void testParseMultipleRow() throws Exception { + + String csv = "value1,c,10,10.1\n" + + "value2,c2,102,true"; + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, new ParseOptions()); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", new Row(new Object[]{"value1", "c", "10", "10.1"}), iterator.next()); + + Assert.assertEquals("Failed to detect 2nd row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 2nd row!", new Row(new Object[]{"value2", "c2", "102", Boolean.TRUE.toString()}), iterator.next()); + + Assert.assertEquals("Failed to detect end of rows!", false, iterator.hasNext()); + Assert.assertEquals("Failed to detect end of rows 2nd time!", false, iterator.hasNext()); + } + } + + @Test + public void testQuotedAndEscapedEndline() throws Exception { + + String csv = "\"row1-\ncol1\",1,1.1\n\"row2-\\\ncol1\",2,2.2\n"; + ParseOptions po = new ParseOptions(); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"row1-\ncol1", "1", "1.1"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + + Row row2 = new Row(new Object[]{"row2-\ncol1", "2", "2.2"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row2, iterator.next()); + + } + } + + @Test + public void testQuotedDoubleQuote() throws Exception { + + String csv = "\"aaa\",\"b\"\"bb\",\"ccc\""; + ParseOptions po = new ParseOptions(); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"aaa", "b\"bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + @Test + public void testEscapedDoubleQuote() throws Exception { + + String csv = "\"aaa\",\"b\\\"bb\",\"ccc\""; + ParseOptions po = new ParseOptions(); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"aaa", "b\"bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + + @Test + public void testSpecialEscape() throws Exception { + + String csv = "\"aaa\",\"b$\"bb\",\"ccc\""; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'$'); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"aaa", "b\"bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + @Test + public void testSpecialEscapedEscape() throws Exception { + + String csv = "aaa,b$$bb,ccc"; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'$'); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"aaa", "b$bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + + @Test + public void testSpecialUnEscapedEscape() throws Exception { + + String csv = "aaa,b$bb,ccc"; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'$'); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Row row = new Row(new Object[]{"aaa", "bbb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + @Test + public void test001Escape() throws Exception { + + String csv = "aaa,b\001\"bb,ccc"; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'\001'); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + Row row = new Row(new Object[]{"aaa", "b\"bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); } + } + + @Test + public void testSpecialQuote() throws Exception { + + String csv = "\001aaa\001,\001b\001\001bb\001,\001ccc\001"; + ParseOptions po = new ParseOptions(); + po.setOption(ParseOptions.OPTIONS_CSV_QUOTE,'\001'); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + Row row = new Row(new Object[]{"aaa", "b\001bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } + + @Test + public void testSpaceAsDelimiterAndQuoted() throws Exception { + + String csv = "aaa \"b bb\" ccc\naaa2 bbb2 \"c cc2\""; + ParseOptions po = new ParseOptions(); +// po.setOption(ParseOptions.OPTIONS_CSV_ESCAPE_CHAR,'\001'); + po.setOption(ParseOptions.OPTIONS_CSV_DELIMITER,' '); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + Row row = new Row(new Object[]{"aaa", "b bb", "ccc"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + + Row row2 = new Row(new Object[]{"aaa2", "bbb2", "c cc2"}); + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row2, iterator.next()); + } + } + + @Test + public void testFailedDelimiterEscaped() throws Exception { + + String csv = "aaa,b\\,bb,ccc"; + ParseOptions po = new ParseOptions(); + + try( + StringReader sr = new StringReader(csv); + OpenCSVParser jp = new OpenCSVParser(sr, po); + ) { + + Iterator<Row> iterator = jp.iterator(); + Row row = new Row(new Object[]{"aaa", "b","bb", "ccc"}); // different from Common CSVParser + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", row, iterator.next()); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/OpenCSVTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/OpenCSVTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/OpenCSVTest.java new file mode 100644 index 0000000..be482c3 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/OpenCSVTest.java @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import com.opencsv.CSVParser; +import com.opencsv.CSVReader; +import com.opencsv.CSVWriter; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +public class OpenCSVTest { + + /** + * no exception in creating csvParser with emtpy stream + * + * @throws IOException + */ + @Test + public void testEmptyStream() throws Exception { + String csv = ""; + + CSVParser jp = new CSVParser(); + String[] columns = jp.parseLine(csv); + Assert.assertEquals("Should detect one column.", 1, columns.length); + Assert.assertEquals("Should detect one column with empty value.", new String[]{""}, columns); + } + + /** + * in case of csv an empty line is still considered as row + * + * @throws IOException + */ + @Test + public void testEmptyRow() throws Exception { + String csv = " "; + CSVParser jp = new CSVParser(); + + String[] columns = jp.parseLine(csv); + Assert.assertEquals("One column not detected.", 1, columns.length); + Assert.assertArrayEquals("Row should not be empty", new String[]{" "}, columns); + } + + @Test + public void testParse1Row() throws Exception { + String csv = "value1,c,10,10.1"; + + String[] cols = csv.split(","); + CSVParser jp = new CSVParser(); + String[] columns = jp.parseLine(csv); + Assert.assertEquals("4 columns not detect", 4, columns.length); + Assert.assertArrayEquals("Row not equal!", cols, columns); + } + + @Test + public void testParseMultipleRow() throws Exception { + + String csv = "value1,c,10,10.1\n" + + "value2,c2,102,true"; + + try( + StringReader sr = new StringReader(csv); + CSVReader csvReader = new CSVReader(sr,',','"','\\'); + ) { + String[] row1 = csvReader.readNext(); + String[] row2 = csvReader.readNext(); + + Assert.assertArrayEquals("Failed to match 1st row!",new String[]{"value1", "c", "10", "10.1"}, row1); + + Assert.assertArrayEquals("Failed to match 2nd row!",new String[]{"value2", "c2", "102", "true"}, row2); + } + } + + @Test + public void testParseCustomSeparator() throws Exception { + + String csv = "value1#c#10#10.1\n" + + "value2#c2#102#true"; + + try( + StringReader sr = new StringReader(csv); + CSVReader csvReader = new CSVReader(sr,'#','"','\\'); + ) { + String[] row1 = csvReader.readNext(); + String[] row2 = csvReader.readNext(); + + Assert.assertArrayEquals("Failed to match 1st row!",new String[]{"value1", "c", "10", "10.1"}, row1); + + Assert.assertArrayEquals("Failed to match 2nd row!",new String[]{"value2", "c2", "102", "true"}, row2); + } + } + + + @Test + public void testParseCustomSeparatorAndQuote() throws Exception { + + String csv = "\"valu#e1\"#c#10#10.1\n" + + "value2#c2#102#true"; + + try( + StringReader sr = new StringReader(csv); + CSVReader csvReader = new CSVReader(sr,'#','"','\\'); + ) { + String[] row1 = csvReader.readNext(); + String[] row2 = csvReader.readNext(); + + Assert.assertArrayEquals("Failed to match 1st row!",new String[]{"valu#e1", "c", "10", "10.1"}, row1); + + Assert.assertArrayEquals("Failed to match 2nd row!",new String[]{"value2", "c2", "102", "true"}, row2); + } + } + + @Test + public void testParseCustomSeparatorAndCustomQuote() throws Exception { + + String csv = "\'valu#e1\'#c#10#10.1\n" + + "value2#c2#102#true"; + + try( + StringReader sr = new StringReader(csv); + CSVReader csvReader = new CSVReader(sr,'#','\'','\\'); + ) { + String[] row1 = csvReader.readNext(); + String[] row2 = csvReader.readNext(); + String[] row3 = csvReader.readNext(); + + Assert.assertArrayEquals("Failed to match 1st row!",new String[]{"valu#e1", "c", "10", "10.1"}, row1); + + Assert.assertArrayEquals("Failed to match 2nd row!",new String[]{"value2", "c2", "102", "true"}, row2); + + Assert.assertArrayEquals("should match Null", null, row3); + } + } + + @Test + public void testWriter() throws Exception { + + String csv = "\'valu#e1\'#c#10#10.1\n" + + "value2#c2#102#true"; + + try( + StringReader sr = new StringReader(csv); + CSVReader csvReader = new CSVReader(sr,'#','\'','\\'); + StringWriter sw = new StringWriter(); + CSVWriter csvWriter = new CSVWriter(sw); + ) { + String[] row1 = csvReader.readNext(); + csvWriter.writeNext(row1); + String[] row2 = csvReader.readNext(); + csvWriter.writeNext(row2); + + Assert.assertEquals("CSVWriter failed.","\"valu#e1\",\"c\",\"10\",\"10.1\"\n" + + "\"value2\",\"c2\",\"102\",\"true\"\n", sw.getBuffer().toString()); + } + } + + @Test + public void testWriterCustomSeparator() throws Exception { + + String csv = "\'valu#e1\'#c#10#10.1\n" + + "value2#c2#102#true"; + + try( + StringReader sr = new StringReader(csv); + CSVReader csvReader = new CSVReader(sr,'#','\'','\\'); + StringWriter sw = new StringWriter(); + CSVWriter csvWriter = new CSVWriter(sw,'$'); + ) { + String[] row1 = csvReader.readNext(); + csvWriter.writeNext(row1); + String[] row2 = csvReader.readNext(); + csvWriter.writeNext(row2); + + Assert.assertEquals("CSVWriter failed.","\"valu#e1\"$\"c\"$\"10\"$\"10.1\"\n" + + "\"value2\"$\"c2\"$\"102\"$\"true\"\n", sw.getBuffer().toString()); + } + } + + @Test + public void testWriterCustomSeparatorAndEnline() throws Exception { + + String csv = "value1,c,10,10.1\n" + + "value2,c2,102,true"; + + try( + StringReader sr = new StringReader(csv); + CSVReader csvReader = new CSVReader(sr,',','\'','\\'); + StringWriter sw = new StringWriter(); + CSVWriter csvWriter = new CSVWriter(sw,'\002',',',"\003"); + ) { + String[] row1 = csvReader.readNext(); + csvWriter.writeNext(row1,false); + String[] row2 = csvReader.readNext(); + csvWriter.writeNext(row2,false); + + Assert.assertEquals("CSVWriter failed.","value1\002c\00210\00210.1\003" + + "value2\002c2\002102\002true\003", sw.getBuffer().toString()); + } + } + + @Test + public void testWriterQuote() throws Exception { + + String csv = "val#ue1,c,10,10.1\n" + + "'val,ue2',c2,102,true\n" + + "val\002ue3,c\0033,103,false"; + + try( + StringReader sr = new StringReader(csv); + CSVReader csvReader = new CSVReader(sr,',','\'','\\'); + StringWriter sw = new StringWriter(); + CSVWriter csvWriter = new CSVWriter(sw,'\002','\'',"\003"); + ) { + String[] row1 = csvReader.readNext(); + csvWriter.writeNext(row1,false); + String[] row2 = csvReader.readNext(); + csvWriter.writeNext(row2,false); + String[] row3 = csvReader.readNext(); + csvWriter.writeNext(row3,false); + + Assert.assertEquals("CSVWriter failed.","val#ue1\u0002c\u000210\u000210.1\u0003" + + "val,ue2\u0002c2\u0002102\u0002true\u0003" + + "'val\u0002ue3'\u0002c\u00033\u0002103\u0002false\u0003", sw.getBuffer().toString()); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java index f549206..1613b05 100644 --- a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.view.hive2.resources.upload; +import org.apache.ambari.view.hive.resources.uploads.query.RowFormat; import org.apache.ambari.view.hive2.client.ColumnDescription; import org.apache.ambari.view.hive2.resources.uploads.ColumnDescriptionImpl; import org.apache.ambari.view.hive2.resources.uploads.HiveFileType; @@ -42,10 +43,12 @@ public class QueryGeneratorTest { cdl.add(new ColumnDescriptionImpl("col4", ColumnDescription.DataTypes.VARCHAR.toString(), 3, 40)); cdl.add(new ColumnDescriptionImpl("col5", ColumnDescription.DataTypes.INT.toString(), 4)); - TableInfo ti = new TableInfo("databaseName", "tableName", cdl, HiveFileType.TEXTFILE); + TableInfo ti = new TableInfo("databaseName", "tableName", cdl, HiveFileType.TEXTFILE, new RowFormat(',', '\\')); QueryGenerator qg = new QueryGenerator(); - Assert.assertEquals("Create query for text file not correct ","create table tableName (col1 CHAR(10), col2 STRING, col3 DECIMAL(10,5), col4 VARCHAR(40), col5 INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE;",qg.generateCreateQuery(ti)); + Assert.assertEquals("Create query for text file not correct ","CREATE TABLE tableName (col1 CHAR(10), col2 STRING," + + " col3 DECIMAL(10,5), col4 VARCHAR(40), col5 INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + + " ESCAPED BY '\\\\' STORED AS TEXTFILE;",qg.generateCreateQuery(ti)); } @Test @@ -58,19 +61,40 @@ public class QueryGeneratorTest { cdl.add(new ColumnDescriptionImpl("col4", ColumnDescription.DataTypes.VARCHAR.toString(), 3, 40)); cdl.add(new ColumnDescriptionImpl("col5", ColumnDescription.DataTypes.INT.toString(), 4)); - TableInfo ti = new TableInfo("databaseName", "tableName", cdl, HiveFileType.ORC); + TableInfo ti = new TableInfo("databaseName", "tableName", cdl, HiveFileType.ORC, new RowFormat(',', '\\')); QueryGenerator qg = new QueryGenerator(); - Assert.assertEquals("Create query for text file not correct ","create table tableName (col1 CHAR(10), col2 STRING, col3 DECIMAL(10,5), col4 VARCHAR(40), col5 INT) STORED AS ORC;",qg.generateCreateQuery(ti)); + Assert.assertEquals("Create query for text file not correct ","CREATE TABLE tableName (col1 CHAR(10), col2 STRING, col3 DECIMAL(10,5), col4 VARCHAR(40), col5 INT) STORED AS ORC;",qg.generateCreateQuery(ti)); } @Test - public void testInsertFromQuery() { + public void testInsertWithoutUnhexFromQuery() { + List<ColumnDescriptionImpl> cdl = new ArrayList<>(4); + cdl.add(new ColumnDescriptionImpl("col1", ColumnDescription.DataTypes.CHAR.toString(), 0, 10)); + cdl.add(new ColumnDescriptionImpl("col2", ColumnDescription.DataTypes.STRING.toString(), 1)); + cdl.add(new ColumnDescriptionImpl("col3", ColumnDescription.DataTypes.DECIMAL.toString(), 2, 10, 5)); + cdl.add(new ColumnDescriptionImpl("col4", ColumnDescription.DataTypes.VARCHAR.toString(), 3, 40)); + cdl.add(new ColumnDescriptionImpl("col5", ColumnDescription.DataTypes.INT.toString(), 4)); + + InsertFromQueryInput ifqi = new InsertFromQueryInput("fromDB","fromTable","toDB","toTable", cdl, Boolean.FALSE); + + QueryGenerator qg = new QueryGenerator(); + Assert.assertEquals("insert from one table to another not correct ","INSERT INTO TABLE toDB.toTable SELECT col1, col2, col3, col4, col5 FROM fromDB.fromTable;",qg.generateInsertFromQuery(ifqi)); + } + + @Test + public void testInsertWithUnhexFromQuery() { + List<ColumnDescriptionImpl> cdl = new ArrayList<>(4); + cdl.add(new ColumnDescriptionImpl("col1", ColumnDescription.DataTypes.CHAR.toString(), 0, 10)); + cdl.add(new ColumnDescriptionImpl("col2", ColumnDescription.DataTypes.STRING.toString(), 1)); + cdl.add(new ColumnDescriptionImpl("col3", ColumnDescription.DataTypes.DECIMAL.toString(), 2, 10, 5)); + cdl.add(new ColumnDescriptionImpl("col4", ColumnDescription.DataTypes.VARCHAR.toString(), 3, 40)); + cdl.add(new ColumnDescriptionImpl("col5", ColumnDescription.DataTypes.INT.toString(), 4)); - InsertFromQueryInput ifqi = new InsertFromQueryInput("fromDB","fromTable","toDB","toTable"); + InsertFromQueryInput ifqi = new InsertFromQueryInput("fromDB","fromTable","toDB","toTable", cdl, Boolean.TRUE); QueryGenerator qg = new QueryGenerator(); - Assert.assertEquals("insert from one table to another not correct ","insert into table toDB.toTable select * from fromDB.fromTable",qg.generateInsertFromQuery(ifqi)); + Assert.assertEquals("insert from one table to another not correct ","INSERT INTO TABLE toDB.toTable SELECT UNHEX(col1), UNHEX(col2), col3, UNHEX(col4), col5 FROM fromDB.fromTable;",qg.generateInsertFromQuery(ifqi)); } @Test @@ -79,6 +103,6 @@ public class QueryGeneratorTest { DeleteQueryInput deleteQueryInput = new DeleteQueryInput("dbName","tableName"); QueryGenerator qg = new QueryGenerator(); - Assert.assertEquals("drop table query not correct ","drop table dbName.tableName",qg.generateDropTableQuery(deleteQueryInput )); + Assert.assertEquals("drop table query not correct ","DROP TABLE dbName.tableName;",qg.generateDropTableQuery(deleteQueryInput )); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c5aafc8f/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java index d65cdb0..88e3ced 100644 --- a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java @@ -18,15 +18,18 @@ package org.apache.ambari.view.hive2.resources.upload; +import org.apache.ambari.view.hive2.client.ColumnDescription; import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.ColumnDescriptionImpl; import org.apache.ambari.view.hive2.resources.uploads.TableDataReader; import org.junit.Assert; import org.junit.Test; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; public class TableDataReaderTest { @@ -75,32 +78,31 @@ public class TableDataReaderTest { @Test public void testCSVReader() throws IOException { RowIter rowIter = new RowIter(10,10); + List<ColumnDescriptionImpl> colDescs = new LinkedList<>(); + for(int i = 0 ; i < 10 ; i++ ) { + ColumnDescriptionImpl cd = new ColumnDescriptionImpl("col" + (i+1) , ColumnDescription.DataTypes.STRING.toString(), i); + colDescs.add(cd); + } - TableDataReader tableDataReader = new TableDataReader(rowIter); + TableDataReader tableDataReader = new TableDataReader(rowIter, colDescs, false); - char[] first10 = "0,1,2,3,4,".toCharArray(); + char del = TableDataReader.CSV_DELIMITER; + char[] first10 = {'0', del, '1', del, '2', del, '3', del, '4', del}; char [] buf = new char[10]; tableDataReader.read(buf,0,10); -// System.out.println("first10 : " + Arrays.toString(first10)); -// System.out.println("buf : " + Arrays.toString(buf)); Assert.assertArrayEquals(first10,buf); - - char[] next11 = "5,6,7,8,9\n1".toCharArray(); + char[] next11 = {'5', del, '6', del, '7', del, '8', del, '9', '\n', '1'}; //"5,6,7,8,9\n1".toCharArray(); char [] buf1 = new char[11]; tableDataReader.read(buf1,0,11); -// System.out.println("next11 : " + Arrays.toString(next11)); -// System.out.println("buf1 : " + Arrays.toString(buf1)); Assert.assertArrayEquals(next11,buf1); // read it fully while( tableDataReader.read(buf,0,10) != -1 ); - char [] last10 = "97,98,99\n,".toCharArray(); // last comma is the left over of previous read. -// System.out.println("last10 : " + Arrays.toString(last10)); -// System.out.println("buf : " + Arrays.toString(buf)); + char [] last10 = {'9', '7', del, '9', '8', del, '9', '9', '\n', del}; //"97,98,99\n,".toCharArray(); // last comma is the left over of previous read. Assert.assertArrayEquals(last10,buf); } @@ -109,7 +111,7 @@ public class TableDataReaderTest { public void testEmptyCSVReader() throws IOException { RowIter rowIter = new RowIter(0,0); - TableDataReader tableDataReader = new TableDataReader(rowIter); + TableDataReader tableDataReader = new TableDataReader(rowIter, null, false); char[] first10 = new char [10]; char [] buf = new char[10]; @@ -120,8 +122,6 @@ public class TableDataReaderTest { tableDataReader.read(buf,0,10); -// System.out.println("first10 : " + Arrays.toString(first10)); -// System.out.println("buf : " + Arrays.toString(buf)); Assert.assertArrayEquals(first10,buf); } }
