Repository: ambari Updated Branches: refs/heads/trunk 45338f5e8 -> ab362d068
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs index b7d9011..f334a3d 100644 --- a/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs +++ b/contrib/views/hive-next/src/main/resources/ui/hive-web/app/templates/upload-table.hbs @@ -30,7 +30,7 @@ <!-- Modal content--> <div class="modal-content"> <div class="modal-header"> - <h4 class="modal-title">Upload Progress</h4> + <h4 class="modal-title">{{t "hive.ui.uploadProgress"}}</h4> </div> <div class="modal-body"> <p> @@ -51,60 +51,67 @@ <div> <table class="table data-upload-form pull-left"> <tr> - <td class="data-upload-form-label"><label>Upload from Local</label></td> + <td class="data-upload-form-label"><label>{{t "hive.ui.uploadFromLocal"}}</label></td> <td class="data-upload-form-field"> {{radio-button value='local' checked=uploadSource}}</td> - <td class="data-upload-form-label"><label>Upload from HDFS</label></td> + <td class="data-upload-form-label"><label>{{t "hive.ui.uploadFromHdfs"}}</label></td> <td class="data-upload-form-field">{{radio-button value='hdfs' checked=uploadSource}}</td> </tr> <tr> - <td class="data-upload-form-label"><label>File type</label></td> + <td class="data-upload-form-label"><label>{{t "hive.ui.fileType"}}</label></td> <td class="data-upload-form-field"> {{typeahead-widget content=inputFileTypes optionValuePath="id" optionLabelPath="name" selection=inputFileType - placeholder="Select File Type"}} + placeholder=(t "hive.ui.uploadFromHdfs") + }} </td> {{#if isLocalUpload }} - <td class="data-upload-form-label"><label>Select from local</label></td> - <td class="data-upload-form-field">{{file-upload filesUploaded="filesUploaded"}}</td> + <td class="data-upload-form-label"><label>{{t "hive.ui.selectFromLocal"}}</label></td> + <td class="data-upload-form-field">{{file-upload filesUploaded="filesUploaded" uploadFiles=files}}</td> {{else}} - <td class="data-upload-form-label"><label>HDFS Path</label></td> + <td class="data-upload-form-label"><label>{{t "hive.ui.hdfsPath"}}</label></td> <td class="data-upload-form-field" id="hdfs-param">{{input type="text" class="form-control" placeholder="Enter full HDFS path" value=hdfsPath }} <button style="margin-left: 5px; padding-top: 6px;padding-bottom: 6px; padding-right: 10px; padding-left: 10px;" type="button" {{action "previewFromHdfs"}} {{bind-attr class=":btn :btn-sm :btn-default"}}>{{t "buttons.showPreview"}}</button></td> {{/if}} </tr> - {{#if rows}} + {{#if showPreview}} <tr> - <td class="data-upload-form-label"><label>Database</label></td> + <td class="data-upload-form-label"><label>{{t "hive.words.database"}}</label></td> <td class="data-upload-form-field"> {{typeahead-widget - content=controllers.databases.databases + content=databases optionValuePath="id" optionLabelPath="name" selection=selectedDatabase - placeholder="Select a Database" + placeholder=(t "hive.ui.selectDatabase") }} </td> - <td class="data-upload-form-label"><label>Table name</label></td> + <td class="data-upload-form-label"><label>{{t "hive.ui.tableName"}}</label></td> <td - class="data-upload-form-field">{{input type="text" class="form-control" placeholder="Table Name" value=tableName }}</td> + class="data-upload-form-field"> + {{#validated-text-field inputValue=tableName allowEmpty=false + tooltip=(t "hive.ui.tableNameTooltip") + invalidClass='form-control red-border' validClass='form-control' regex=TABLE_NAME_REGEX + errorMessage=(t "hive.ui.tableNameErrorMessage") }} + {{/validated-text-field}} + </td> </tr> <tr> - <td class="data-upload-form-label"><label>Stored as</label></td> + <td class="data-upload-form-label"><label>{{t "hive.ui.storedAs"}}</label></td> <td class="data-upload-form-field"> {{typeahead-widget content=fileTypes selection=selectedFileType}} </td> {{#if inputFileTypeCSV }} - <td class="data-upload-form-label"><label>Is first row header ?</label></td> + <td class="data-upload-form-label"><label>{{t "hive.ui.isFirstRowHeader"}}</label></td> <td class="data-upload-form-field"> {{input id="isFirstRowHeader" type="checkbox" checked=isFirstRowHeader }} </td> @@ -114,27 +121,32 @@ {{/if}} </table> + {{#if showPreview}} <table class="pull-right"> <tr> <td> - {{#if rows}} <button type="button" {{action "uploadTable"}} {{bind-attr class=":btn :btn-sm :btn-default"}}>{{t "buttons.uploadTable"}}</button> - {{/if}} </td> </tr> </table> - + {{/if}} </div> <div> - {{#if rows}} + {{#if showPreview}} <div id="upload-table"> <table class="table table-expandable no-border"> <thead> <tr> {{#each column in header}} - <th>{{input placeholder="column-name" type="text" class="form-control" value=column.name}}</th> + <th> + {{#validated-text-field inputValue=column.name allowEmpty=false + tooltip=(t "hive.ui.columnNameTooltip") + invalidClass='form-control red-border' validClass='form-control' regex=COLUMN_NAME_REGEX + errorMessage=(t "hive.ui.columnNameErrorMessage")}} + {{/validated-text-field}} + </th> {{/each}} </tr> <tr id="upload-controls"> @@ -165,4 +177,4 @@ </div> {{/if}} </div> -</div> \ No newline at end of file +</div> http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java new file mode 100644 index 0000000..fab1328 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/CSVParserTest.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.parsers.csv.CSVParser; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; + +public class CSVParserTest { + + /** + * no exception in creating csvParser with emtpy stream + * @throws IOException + */ + @Test + public void testEmptyStream() throws Exception { + String csv = ""; + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, null); + ) { + Assert.assertEquals("There should not be any rows.",false, jp.iterator().hasNext()); + } + } + + /** + * in case of csv an empty line is still considered as row + * @throws IOException + */ + @Test + public void testEmptyRow() throws Exception { + String csv = " "; + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, null); + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator should be Empty", true, iterator.hasNext()); + Assert.assertArrayEquals("Row should not be empty",new Object[]{" "},iterator.next().getRow()); + } + } + + @Test + public void testParse1Row() throws Exception { + String csv = "value1,c,10,10.1"; + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, null); + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator Empty!", true, iterator.hasNext()); + Row row = iterator.next(); + Row expected = new Row(new Object[]{"value1", "c", "10", "10.1"}); + Assert.assertEquals("Row not equal!", expected, row); + + Assert.assertEquals("Should report no more rows!", false, iterator.hasNext()); + } + } + + @Test + public void testParseMultipleRow() throws Exception { + + String csv = "value1,c,10,10.1\n" + + "value2,c2,102,true"; + + try( + StringReader sr = new StringReader(csv); + CSVParser jp = new CSVParser(sr, null); + ) { + + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", new Row(new Object[]{"value1", "c", "10", "10.1"}), iterator.next()); + + Assert.assertEquals("Failed to detect 2nd row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 2nd row!", new Row(new Object[]{"value2", "c2", "102", Boolean.TRUE.toString()}), iterator.next()); + + Assert.assertEquals("Failed to detect end of rows!", false, iterator.hasNext()); + Assert.assertEquals("Failed to detect end of rows 2nd time!", false, iterator.hasNext()); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java new file mode 100644 index 0000000..6b57228 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserCSVTest.java @@ -0,0 +1,326 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import org.apache.ambari.view.hive2.client.ColumnDescription; +import org.apache.ambari.view.hive2.client.ColumnDescriptionShort; +import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.ColumnDescriptionImpl; +import org.apache.ambari.view.hive2.resources.uploads.parsers.DataParser; +import org.apache.ambari.view.hive2.resources.uploads.parsers.ParseOptions; +import org.apache.ambari.view.hive2.resources.uploads.parsers.PreviewData; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; + +public class DataParserCSVTest { + @Test + public void testParsePreviewCSV() throws Exception { + String str = "1,a\n" + + "2,b\n" + + "3,c\n"; + + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + + try ( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ){ + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getPreviewRows()); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(2, pd.getPreviewRows().size()); // now it will not return the first row which is header + Assert.assertEquals(2, pd.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("1", ColumnDescriptionShort.DataTypes.INT.toString(), 0), + new ColumnDescriptionImpl("a", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1)}; + + Object cols2[] = new Object[2]; + cols2[0] = "2"; + cols2[1] = "b"; + Row row2 = new Row(cols2); + + Object cols3[] = new Object[2]; + cols3[0] = "3"; + cols3[1] = "c"; + Row row3 = new Row(cols3); + + Row[] rows = { row2, row3}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray()); + } + } + + /** + * even if in one of the preview rows, datatype is not correct, then it should be assigned that datatype. + * but if first row is header then first row should not be acconted for detecting datatype + * @throws IOException + */ + @Test + public void testParsePreviewDataTypeDetectionCSV() throws Exception { + String str = "1,a,10,k\n" + + "2,b,6,8\n" + + "2.2,b,7,9\n" + + "2,b,abc,1\n" + + "2,b,9,3\n" + + "2,b,8,5\n" + + "2,b,7,3\n" + + "2,b,6,3\n" + + "3,c,c,3\n"; + + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + try(StringReader sr = new StringReader(str); + DataParser dp= new DataParser(sr, parseOptions)) { + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(4, pd.getHeader().size()); + ColumnDescription[] cd = { + // as row 3 contains 2.2 + new ColumnDescriptionImpl("1", ColumnDescriptionShort.DataTypes.DOUBLE.toString(), 0), + // as all are chars + new ColumnDescriptionImpl("a", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1), + // as row 4 contains abc + new ColumnDescriptionImpl("10", ColumnDescriptionShort.DataTypes.STRING.toString(), 2), + // although row 1 contains k but it is in header and not counted in detecting datatype + new ColumnDescriptionImpl("k", ColumnDescriptionShort.DataTypes.INT.toString(), 3)}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + } + } + + /** + * even if in one of the preview rows, datatype is not correct, then it should be assigned that datatype. + * but if first row is header then first row should not be acconted for detecting datatype + * @throws IOException + */ + @Test + public void testParsePreviewDataTypeDetection2CSV() throws Exception { + String str = "1,a,10,k\n" + + "2,b,6,p\n" + + "2.2,b,7,9\n" + + "2,b,2.2,1\n" + + "2,b,9,3\n" + + "2,b,8,5\n" + + "2,b,7,3\n" + + "2,b,6,3\n" + + "3,c,c,3\n"; + + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + + try(StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions)) { + + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(4, pd.getHeader().size()); + ColumnDescription[] cd = { + // as row 3 contains 2.2 + new ColumnDescriptionImpl("1", ColumnDescriptionShort.DataTypes.DOUBLE.toString(), 0), + // as all are chars + new ColumnDescriptionImpl("a", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1), + // some are int, char and some double .. nothing other than 'string' satisfies all the rows + new ColumnDescriptionImpl("10", ColumnDescriptionShort.DataTypes.STRING.toString(), 2), + // although row 1 contains k but it is in header and not counted in detecting datatype + // but row 2 also has a char p which will be acconted for datatype detection + new ColumnDescriptionImpl("k", ColumnDescriptionShort.DataTypes.CHAR.toString(), 3)}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + } + } + + /** + * One row csv will give default column names and 1st row in preview if HEADER.PROVIDED_BY_USER is selected + * @throws IOException + */ + @Test + public void testParsePreview1RowCSV() throws Exception { + String str = "1,a\n" ; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.PROVIDED_BY_USER.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getPreviewRows()); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(1, pd.getPreviewRows().size()); + Assert.assertEquals(2, pd.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("Column1", ColumnDescriptionShort.DataTypes.INT.toString(), 0), + new ColumnDescriptionImpl("Column2", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1)}; + + Object cols1[] = new Object[2]; + cols1[0] = "1"; + cols1[1] = "a"; + Row row1 = new Row(cols1); + + Row[] rows = {row1}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray()); + } + } + + /** + * One row csv will throw exception in preview if HEADER.FIRST_RECORD is selected. + * @throws IOException + */ + @Test(expected = java.util.NoSuchElementException.class) + public void testParsePreview1RowCSVFirstRowHeader() throws Exception { + String str = "col1,col2\n" ; + + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + + PreviewData pd = dp.parsePreview(); + } + } + + /** + * more number of columns in a row => igore the extra columns. Number of columns is decided by the first row. + * If other row contains more columns then those columns will be ignored + * Here first row has 2 columns and second row has 3 columns so the value 'x' is ignored + * @throws IOException + */ + @Test + public void testParsePreviewCSVMoreColumns() throws Exception { + String str = "1,a\n" + + "2,b,x\n" + // contains 3 cols, more number of columns + "3,c\n"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + PreviewData pd = dp.parsePreview(); + Row row = new Row(new Object[]{"2","b"}); + + Assert.assertArrayEquals("Additional columns not properly handled.", row.getRow(),pd.getPreviewRows().get(0).getRow()); + } + } + + /** + * less number of columns => treat missing values as null. Number of columns is decided by the first row of the table + * if other rows has less number of columns then it treats other columns as null + * @throws IOException + */ + @Test + public void testParsePreviewCSVLessColumns() throws Exception { + String str = "1,a\n" + + "2\n" + // contains 1 col, less number of columns + "3,c\n"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertEquals("Missing value not detected as null.",pd.getPreviewRows().get(1).getRow()[1],null); + } + } + + /** + * empty values are treated as empty string + * @throws IOException + */ + @Test + public void testEmptyColumn() throws Exception { + String str = "1,a,x\n" + + "2,,y\n" + // contains 1 col, less number of columns + "3,c,z\n"; + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertEquals("Empty column not detected properly.",pd.getPreviewRows().get(0).getRow()[1],""); + } + } + + /** + * empty values are treated as empty string + * @throws IOException + */ + @Test + public void testLastEmptyColumn() throws Exception { + String str = "1,a,x\n" + + "2,,\n" + // contains 1 col, less number of columns + "3,c,z\n"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertEquals("Empty column not detected properly.",pd.getPreviewRows().get(0).getRow()[1],""); + Assert.assertEquals("Empty column not detected properly.",pd.getPreviewRows().get(0).getRow()[2],""); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java new file mode 100644 index 0000000..1ea7ec2 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserJSONTest.java @@ -0,0 +1,263 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import org.apache.ambari.view.hive2.client.ColumnDescription; +import org.apache.ambari.view.hive2.client.ColumnDescriptionShort; +import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.ColumnDescriptionImpl; +import org.apache.ambari.view.hive2.resources.uploads.parsers.DataParser; +import org.apache.ambari.view.hive2.resources.uploads.parsers.ParseOptions; +import org.apache.ambari.view.hive2.resources.uploads.parsers.PreviewData; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; + +public class DataParserJSONTest { + + @Test + public void testParsePreviewJSON() throws Exception { + String str = "[ {\"col1\" : \"a\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"4.4\" }," + + "{\"col1\": \"b\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"5.4\" }," + + "{\"col1\": \"c\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"6.4\" }," + + "{\"col1\": \"d\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"7.4\" }," + + "{\"col1\": \"e\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"8.4\" }," + + "{\"col1\": \"f\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"9.4\" }," + + "{\"col1\": \"g\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"10.4\" }," + + "{\"col1\": \"h\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"11.4\" }," + + "{\"col1\": \"i\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"4\" }," + + "{\"col1\": \"j\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"5\" }," + + "{\"col1\": \"k\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"6\" }," + + "{\"col1\": \"l\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"7\" }," + + "{\"col1\": \"m\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"24.4\" }," + + "{\"col1\": \"n\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"14.4\" }," + + "{\"col1\": \"o\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"34.4\" }," + + "{\"col1\": \"p\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"44.4\" }," + + "{\"col1\": \"q\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"54.4\" }," + + "{\"col1\": \"r\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"64.4\" }" + + "]"; + + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_NUMBER_OF_PREVIEW_ROWS, 7); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getPreviewRows()); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(7, pd.getPreviewRows().size()); // header row + preview rows + Assert.assertEquals(14, pd.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("col1", ColumnDescriptionShort.DataTypes.CHAR.toString(), 0), + new ColumnDescriptionImpl("col2", ColumnDescriptionShort.DataTypes.STRING.toString(), 1), + new ColumnDescriptionImpl("col3", ColumnDescriptionShort.DataTypes.STRING.toString(), 2), + new ColumnDescriptionImpl("col4", ColumnDescriptionShort.DataTypes.STRING.toString(), 3), + new ColumnDescriptionImpl("col5", ColumnDescriptionShort.DataTypes.STRING.toString(), 4), + new ColumnDescriptionImpl("col6", ColumnDescriptionShort.DataTypes.STRING.toString(), 5), + new ColumnDescriptionImpl("col7", ColumnDescriptionShort.DataTypes.STRING.toString(), 6), + new ColumnDescriptionImpl("col8", ColumnDescriptionShort.DataTypes.STRING.toString(), 7), + new ColumnDescriptionImpl("col9", ColumnDescriptionShort.DataTypes.STRING.toString(), 8), + new ColumnDescriptionImpl("col10", ColumnDescriptionShort.DataTypes.STRING.toString(), 9), + new ColumnDescriptionImpl("col11", ColumnDescriptionShort.DataTypes.STRING.toString(), 10), + new ColumnDescriptionImpl("col12", ColumnDescriptionShort.DataTypes.STRING.toString(), 11), + new ColumnDescriptionImpl("col13", ColumnDescriptionShort.DataTypes.STRING.toString(), 12), + new ColumnDescriptionImpl("col14", ColumnDescriptionShort.DataTypes.DOUBLE.toString(), 13)}; + + Row row2 = new Row(new Object[]{"a", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "4.4"}); + Row row3 = new Row(new Object[]{"b", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "5.4"}); + Row row4 = new Row(new Object[]{"c", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "6.4"}); + Row row5 = new Row(new Object[]{"d", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "7.4"}); + Row row6 = new Row(new Object[]{"e", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "8.4"}); + Row row7 = new Row(new Object[]{"f", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "9.4"}); + Row row8 = new Row(new Object[]{"g", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "10.4"}); + + Row[] rows = { row2, row3, row4, row5, row6, row7, row8}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray()); + } + } + + /** + * additional columns in rows of JSON are ignored. + * + * @throws IOException + */ + @Test + public void testParsePreviewCSVMoreColumns() throws Exception { + String str = "[ {\"col1\" : \"a\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"4.4\" }," + + "{\"col1\": \"b\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" , \"col14\" : \"43.4\" ,\"col15\" : \"asafsfa\" }," + + "{\"col1\": \"c\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"6.4\" }," + + "{\"col1\": \"d\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"7.4\" }" + + "]"; + + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + PreviewData pd = dp.parsePreview(); + + Row row2 = new Row(new Object[]{"b", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "43.4"}); + Assert.assertArrayEquals("More number of columns do not give correct result.", row2.getRow(), pd.getPreviewRows().get(1).getRow()); + } + } + + /** + * less columns in json makes them null. + * + * @throws IOException + */ + @Test + public void testParsePreviewCSVLessColumns() throws Exception { + String str = "[ " + + "{\"col1\" : \"a\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"4.4\" }," + + "{\"col1\": \"b\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" }," + + "{\"col1\": \"c\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"6.4\" }," + + "{\"col1\": \"d\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"7.4\" }" + + "]"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + PreviewData pd = dp.parsePreview(); + + Assert.assertNull(pd.getPreviewRows().get(1).getRow()[13]); + } + } + + /** + * illegal json format gives error + * + * @throws IOException + */ + @Test(expected = IllegalArgumentException.class) + public void testWrongJsonFormat() throws Exception { + String str = "[ " + + "{\"col1\" : \"a\", \n\"col2\": \"abcd\" }," + + "{\"col1\": \"b\", \n\"col2\": \"abcd\" }," + + "{\"col1\": \"c\", \n\"col2\": \"abcd\" }," + + "{\"col1\": \"d\",, \n\"col2\": \"abcd\" }" // extra comma in this line + + "]"; + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ) { + PreviewData pd = dp.parsePreview(); + } + } + + /** + * One row JSON will give embedde column names and 1st row in preview if HEADER.EMBEDDED is selected + * @throws IOException + */ + @Test + public void testParsePreview1RowJSON() throws Exception { + String str = "[ " + + "{\"col1\": \"d\", \n\"col2\": \"abcd\" }" // extra comma in this line + + "]"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getPreviewRows()); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(1, pd.getPreviewRows().size()); + Assert.assertEquals(2, pd.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("col1", ColumnDescriptionShort.DataTypes.CHAR.toString(), 0), + new ColumnDescriptionImpl("col2", ColumnDescriptionShort.DataTypes.STRING.toString(), 1)}; + + Object cols1[] = new Object[2]; + cols1[0] = "d"; + cols1[1] = "abcd"; + Row row1 = new Row(cols1); + + Row[] rows = {row1}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray()); + } + } + + /** + * One row JSON will give default column names and 1st row in preview if HEADER.PROVIDED_BY_USER is selected + * @throws IOException + */ + @Test + public void testParsePreview1RowJSONHeaderProvided() throws Exception { + String str = "[ " + + "{\"col1\": \"d\", \n\"col2\": \"abcd\" }" // extra comma in this line + + "]"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.PROVIDED_BY_USER.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getPreviewRows()); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(1, pd.getPreviewRows().size()); + Assert.assertEquals(2, pd.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("Column1", ColumnDescriptionShort.DataTypes.CHAR.toString(), 0), + new ColumnDescriptionImpl("Column2", ColumnDescriptionShort.DataTypes.STRING.toString(), 1)}; + + Object cols1[] = new Object[2]; + cols1[0] = "d"; + cols1[1] = "abcd"; + Row row1 = new Row(cols1); + + Row[] rows = {row1}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray()); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java new file mode 100644 index 0000000..2faf4e9 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/DataParserXMLTest.java @@ -0,0 +1,295 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import org.apache.ambari.view.hive2.client.ColumnDescription; +import org.apache.ambari.view.hive2.client.ColumnDescriptionShort; +import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.ColumnDescriptionImpl; +import org.apache.ambari.view.hive2.resources.uploads.parsers.DataParser; +import org.apache.ambari.view.hive2.resources.uploads.parsers.ParseOptions; +import org.apache.ambari.view.hive2.resources.uploads.parsers.PreviewData; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; + +public class DataParserXMLTest { + + @Test + public void testParsePreviewXML() throws Exception { + String str = "<table>" + + "<row>" + + "<col name=\"col1\">row1-col1-Value</col>" + + "<col name=\"col2\">row1-col2-Value</col>" + + "<col name=\"col3\">row1-col3-Value</col>" + + "<col name=\"col4\">10</col>" + + "<col name=\"col5\">11</col>" + + "</row>" + + "<row>" + + "<col name=\"col1\">row2-col1-Value</col>" + + "<col name=\"col2\">row2-col2-Value</col>" + + "<col name=\"col3\">row2-col3-Value</col>" + + "<col name=\"col4\">20</col>" + + "<col name=\"col5\">21</col>" + + "</row>" + + "</table>"; + + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getPreviewRows()); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(2, pd.getPreviewRows().size()); // header row + preview rows + Assert.assertEquals(5, pd.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("col1", ColumnDescriptionShort.DataTypes.STRING.toString(), 0), + new ColumnDescriptionImpl("col2", ColumnDescriptionShort.DataTypes.STRING.toString(), 1), + new ColumnDescriptionImpl("col3", ColumnDescriptionShort.DataTypes.STRING.toString(), 2), + new ColumnDescriptionImpl("col4", ColumnDescriptionShort.DataTypes.INT.toString(), 3), + new ColumnDescriptionImpl("col5", ColumnDescriptionShort.DataTypes.INT.toString(), 4) + }; + + Row row2 = new Row(new Object[]{"row1-col1-Value", "row1-col2-Value", "row1-col3-Value", "10", "11"}); + Row row3 = new Row(new Object[]{"row2-col1-Value", "row2-col2-Value", "row2-col3-Value", "20", "21"}); + + Row[] rows = {row2, row3}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray()); + } + } + + + /** + * additional columns in rows of XML are ignored. + * number of columns are decided by the first row of the table and here second row contains more columns so those are ignored. + * @throws IOException + */ + @Test + public void testParsePreviewCSVMoreColumns() throws Exception { + String str ="<table>" + + "<row>" + + "<col name=\"col1\">row1-col1-Value</col>" + + "<col name=\"col2\">row1-col2-Value</col>" + + "<col name=\"col3\">row1-col3-Value</col>" + + "<col name=\"col4\">10</col>" + + "<col name=\"col5\">11</col>" + + "</row>" + + "<row>" + + "<col name=\"col1\">row2-col1-Value</col>" + + "<col name=\"col2\">row2-col2-Value</col>" + + "<col name=\"col3\">row2-col3-Value</col>" + + "<col name=\"col99\">row2-col99-Value</col>" + // extra colummn + "<col name=\"col100\">row2-col100-Value</col>" + // extra column + "<col name=\"col4\">20</col>" + + "<col name=\"col5\">21</col>" + + "</row>" + + "</table>"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString()); + + + try( StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ) { + + + PreviewData pd = dp.parsePreview(); + + Row row2 = new Row(new Object[]{"row2-col1-Value","row2-col2-Value","row2-col3-Value","20","21"}); + Assert.assertArrayEquals("More number of columns do not give correct result.", row2.getRow(), pd.getPreviewRows().get(1).getRow()); + } + } + + /** + * less columns in xml makes them null. + * number of columns are decided by the first row of the table and here second row does not contain col99 and col100 + * columns so those are set to null. + * @throws IOException + */ + @Test + public void testParsePreviewCSVLessColumns() throws Exception { + String str = "<table>" + + "<row>" + + "<col name=\"col1\">row1-col1-Value</col>" + + "<col name=\"col2\">row1-col2-Value</col>" + + "<col name=\"col3\">row1-col3-Value</col>" + + "<col name=\"col99\">row2-col99-Value</col>" + // extra colummn + "<col name=\"col100\">row2-col100-Value</col>" + // extra column + "<col name=\"col4\">10</col>" + + "<col name=\"col5\">11</col>" + + "</row>" + + "<row>" + + "<col name=\"col1\">row2-col1-Value</col>" + + "<col name=\"col2\">row2-col2-Value</col>" + + "<col name=\"col3\">row2-col3-Value</col>" + + "<col name=\"col4\">20</col>" + + "<col name=\"col5\">21</col>" + + "</row>" + + "</table>"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ) { + PreviewData pd = dp.parsePreview(); + + Row row2 = new Row(new Object[]{"row2-col1-Value","row2-col2-Value","row2-col3-Value",null,null,"20","21"}); + Assert.assertArrayEquals("Less number of columns do not give correct result.", row2.getRow(), pd.getPreviewRows().get(1).getRow()); + } + } + + /** + * illegal xml format gives error. adding illegal tag gives error + * + * @throws IOException + */ + @Test(expected = IllegalArgumentException.class) + public void testWrongXMLFormat() throws Exception { + String str = "<table>" + + "<row>" + + "<ccc></ccc>" + // illegal tag. + "<col name=\"col1\">row1-col1-Value</col>" + + "<col name=\"col2\">row1-col2-Value</col>" + + "<col name=\"col3\">row1-col3-Value</col>" + + "<col name=\"col99\">row2-col99-Value</col>" + // extra colummn + "<col name=\"col100\">row2-col100-Value</col>" + // extra column + "<col name=\"col4\">10</col>" + + "<col name=\"col5\">11</col>" + + "</row>" + + "<row>" + + "<col name=\"col1\">row2-col1-Value</col>" + + "<col name=\"col2\">row2-col2-Value</col>" + + "<col name=\"col3\">row2-col3-Value</col>" + + "<col name=\"col4\">20</col>" + + "<col name=\"col5\">21</col>" + + "</row>" + + "</table>"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString()); + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ) { + PreviewData pd = dp.parsePreview(); + } + } + + /** + * One row XML will give embedde column names and 1st row in preview if HEADER.EMBEDDED is selected + * @throws IOException + */ + @Test + public void testParsePreview1RowXML() throws Exception { + String str = "<table>" + + "<row>" + + "<col name=\"col1\">row1-col1-Value</col>" + + "<col name=\"col2\">11</col>" + + "</row>" + + "</table>"; + + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions); + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getPreviewRows()); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(1, pd.getPreviewRows().size()); + Assert.assertEquals(2, pd.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("col1", ColumnDescriptionShort.DataTypes.STRING.toString(), 0), + new ColumnDescriptionImpl("col2", ColumnDescriptionShort.DataTypes.INT.toString(), 1)}; + + Object cols1[] = new Object[2]; + cols1[0] = "row1-col1-Value"; + cols1[1] = "11"; + Row row1 = new Row(cols1); + + Row[] rows = {row1}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray()); + } + } + + /** + * One row XML will give default column names and 1st row in preview if HEADER.PROVIDED_BY_USER is selected + * @throws IOException + */ + @Test + public void testParsePreview1RowXMLHeaderProvided() throws Exception { + String str = "<table>" + + "<row>" + + "<col name=\"col1\">row1-col1-Value</col>" + + "<col name=\"col2\">11</col>" + + "</row>" + + "</table>"; + + ParseOptions parseOptions = new ParseOptions(); + parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString()); + parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.PROVIDED_BY_USER.toString()); + + try( + StringReader sr = new StringReader(str); + DataParser dp = new DataParser(sr, parseOptions) + ) { + + PreviewData pd = dp.parsePreview(); + Assert.assertNotNull(pd.getPreviewRows()); + Assert.assertNotNull(pd.getHeader()); + Assert.assertEquals(1, pd.getPreviewRows().size()); + Assert.assertEquals(2, pd.getHeader().size()); + ColumnDescription[] cd = {new ColumnDescriptionImpl("Column1", ColumnDescriptionShort.DataTypes.STRING.toString(), 0), + new ColumnDescriptionImpl("Column2", ColumnDescriptionShort.DataTypes.INT.toString(), 1)}; + + Object cols1[] = new Object[2]; + cols1[0] = "row1-col1-Value"; + cols1[1] = "11"; + Row row1 = new Row(cols1); + + Row[] rows = {row1}; + + Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray()); + Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray()); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/JsonParserTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/JsonParserTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/JsonParserTest.java new file mode 100644 index 0000000..b5e3a93 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/JsonParserTest.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.parsers.json.JSONParser; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; + +public class JsonParserTest { + + @Test(expected = IOException.class) + public void testEmptyStream() throws Exception { + String json = ""; + + try( + StringReader sr = new StringReader(json); + JSONParser jp = new JSONParser(sr, null); + ) { + // PARSING WILL THROW ERROR + } + } + + @Test + public void testEmptyRow() throws Exception { + JsonObject jo = new JsonObject(); + JsonArray ja = new JsonArray(); + ja.add(jo); + String json = ja.toString(); + + try( + StringReader sr = new StringReader(json); + JSONParser jp = new JSONParser(sr, null) + ) { + + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator should not be Empty", true, iterator.hasNext()); + Assert.assertArrayEquals("Row should be empty",new Object[]{},iterator.next().getRow()); + } + } + + + @Test + public void testEmptyTable() throws Exception { + JsonArray ja = new JsonArray(); + String json = ja.toString(); + + try( + StringReader sr = new StringReader(json); + JSONParser jp = new JSONParser(sr, null); + ) { + Iterator<Row> iterator = jp.iterator(); + Assert.assertEquals("Iterator Empty!", false, iterator.hasNext()); + } + } + + @Test + public void testParse1Row() throws Exception { + JsonObject jo = new JsonObject(); + jo.addProperty("key1","value1"); + jo.addProperty("key2",'c'); + jo.addProperty("key3",10); + jo.addProperty("key4",10.1); + + JsonArray ja = new JsonArray(); + ja.add(jo); + String json = ja.toString(); + + try(StringReader sr = new StringReader(json); + + JSONParser jp = new JSONParser(sr, null) + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator Empty!", true, iterator.hasNext()); + Row row = iterator.next(); + Row expected = new Row(new Object[]{"value1", "c", "10", "10.1"}); + Assert.assertEquals("Row not equal!", expected, row); + + Assert.assertEquals("Should report no more rows!", false, iterator.hasNext()); + } + } + + @Test + public void testParseMultipleRow() throws Exception { + JsonObject jo1 = new JsonObject(); + jo1.addProperty("key1","value1"); + jo1.addProperty("key2","c"); + jo1.addProperty("key3","10"); + jo1.addProperty("key4","10.1"); + + JsonObject jo2 = new JsonObject(); + jo2.addProperty("key1","value2"); + jo2.addProperty("key2","c2"); + jo2.addProperty("key3","102"); + jo2.addProperty("key4",true); + + + JsonArray ja = new JsonArray(); + ja.add(jo1); + ja.add(jo2); + + String json = ja.toString(); + + + + try( + StringReader sr = new StringReader(json); + JSONParser jp = new JSONParser(sr, null) + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", new Row(new Object[]{"value1", "c", "10", "10.1"}), iterator.next()); + + Assert.assertEquals("Failed to detect 2nd row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 2nd row!", new Row(new Object[]{"value2", "c2", "102", Boolean.TRUE.toString()}), iterator.next()); + + Assert.assertEquals("Failed to detect end of rows!", false, iterator.hasNext()); + Assert.assertEquals("Failed to detect end of rows 2nd time!", false, iterator.hasNext()); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java new file mode 100644 index 0000000..f549206 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/QueryGeneratorTest.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import org.apache.ambari.view.hive2.client.ColumnDescription; +import org.apache.ambari.view.hive2.resources.uploads.ColumnDescriptionImpl; +import org.apache.ambari.view.hive2.resources.uploads.HiveFileType; +import org.apache.ambari.view.hive2.resources.uploads.query.DeleteQueryInput; +import org.apache.ambari.view.hive2.resources.uploads.query.InsertFromQueryInput; +import org.apache.ambari.view.hive2.resources.uploads.query.QueryGenerator; +import org.apache.ambari.view.hive2.resources.uploads.query.TableInfo; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class QueryGeneratorTest { + @Test + public void testCreateTextFile() { + + List<ColumnDescriptionImpl> cdl = new ArrayList<>(4); + cdl.add(new ColumnDescriptionImpl("col1", ColumnDescription.DataTypes.CHAR.toString(), 0, 10)); + cdl.add(new ColumnDescriptionImpl("col2", ColumnDescription.DataTypes.STRING.toString(), 1)); + cdl.add(new ColumnDescriptionImpl("col3", ColumnDescription.DataTypes.DECIMAL.toString(), 2, 10, 5)); + cdl.add(new ColumnDescriptionImpl("col4", ColumnDescription.DataTypes.VARCHAR.toString(), 3, 40)); + cdl.add(new ColumnDescriptionImpl("col5", ColumnDescription.DataTypes.INT.toString(), 4)); + + TableInfo ti = new TableInfo("databaseName", "tableName", cdl, HiveFileType.TEXTFILE); + + QueryGenerator qg = new QueryGenerator(); + Assert.assertEquals("Create query for text file not correct ","create table tableName (col1 CHAR(10), col2 STRING, col3 DECIMAL(10,5), col4 VARCHAR(40), col5 INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE;",qg.generateCreateQuery(ti)); + } + + @Test + public void testCreateORC() { + + List<ColumnDescriptionImpl> cdl = new ArrayList<>(4); + cdl.add(new ColumnDescriptionImpl("col1", ColumnDescription.DataTypes.CHAR.toString(), 0, 10)); + cdl.add(new ColumnDescriptionImpl("col2", ColumnDescription.DataTypes.STRING.toString(), 1)); + cdl.add(new ColumnDescriptionImpl("col3", ColumnDescription.DataTypes.DECIMAL.toString(), 2, 10, 5)); + cdl.add(new ColumnDescriptionImpl("col4", ColumnDescription.DataTypes.VARCHAR.toString(), 3, 40)); + cdl.add(new ColumnDescriptionImpl("col5", ColumnDescription.DataTypes.INT.toString(), 4)); + + TableInfo ti = new TableInfo("databaseName", "tableName", cdl, HiveFileType.ORC); + + QueryGenerator qg = new QueryGenerator(); + Assert.assertEquals("Create query for text file not correct ","create table tableName (col1 CHAR(10), col2 STRING, col3 DECIMAL(10,5), col4 VARCHAR(40), col5 INT) STORED AS ORC;",qg.generateCreateQuery(ti)); + } + + @Test + public void testInsertFromQuery() { + + InsertFromQueryInput ifqi = new InsertFromQueryInput("fromDB","fromTable","toDB","toTable"); + + QueryGenerator qg = new QueryGenerator(); + Assert.assertEquals("insert from one table to another not correct ","insert into table toDB.toTable select * from fromDB.fromTable",qg.generateInsertFromQuery(ifqi)); + } + + @Test + public void testDropTableQuery() { + + DeleteQueryInput deleteQueryInput = new DeleteQueryInput("dbName","tableName"); + + QueryGenerator qg = new QueryGenerator(); + Assert.assertEquals("drop table query not correct ","drop table dbName.tableName",qg.generateDropTableQuery(deleteQueryInput )); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java new file mode 100644 index 0000000..d65cdb0 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/TableDataReaderTest.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.TableDataReader; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; + +public class TableDataReaderTest { + + private class RowIter implements Iterator<Row> { + int numberOfRows; + int numberOfCols; + int index = 0 ; + ArrayList<Row> rows = new ArrayList<Row>(); + public RowIter(int numberOfRows, int numberOfCols){ + this.numberOfRows = numberOfRows; + this.numberOfCols = numberOfCols; + int x = 0 ; + for(int i = 0; i < this.numberOfRows; i++ ){ + Object [] objArray = new Object[10]; + for(int j = 0; j < this.numberOfCols; j++ ){ + objArray[j] = x++ + "" ; + } + Row row = new Row(objArray); + rows.add(row); + } + } + @Override + public boolean hasNext() { + return index < numberOfRows; + } + + @Override + public Row next() { + return rows.get(index++); + } + + @Override + public void remove() { + throw new RuntimeException("Operation not supported."); + } + + @Override + public String toString() { + return "RowIter{" + + "index=" + index + + ", rows=" + rows + + '}'; + } + } + + @Test + public void testCSVReader() throws IOException { + RowIter rowIter = new RowIter(10,10); + + TableDataReader tableDataReader = new TableDataReader(rowIter); + + char[] first10 = "0,1,2,3,4,".toCharArray(); + char [] buf = new char[10]; + tableDataReader.read(buf,0,10); + +// System.out.println("first10 : " + Arrays.toString(first10)); +// System.out.println("buf : " + Arrays.toString(buf)); + Assert.assertArrayEquals(first10,buf); + + + char[] next11 = "5,6,7,8,9\n1".toCharArray(); + char [] buf1 = new char[11]; + tableDataReader.read(buf1,0,11); + +// System.out.println("next11 : " + Arrays.toString(next11)); +// System.out.println("buf1 : " + Arrays.toString(buf1)); + Assert.assertArrayEquals(next11,buf1); + + // read it fully + while( tableDataReader.read(buf,0,10) != -1 ); + + char [] last10 = "97,98,99\n,".toCharArray(); // last comma is the left over of previous read. +// System.out.println("last10 : " + Arrays.toString(last10)); +// System.out.println("buf : " + Arrays.toString(buf)); + + Assert.assertArrayEquals(last10,buf); + } + + @Test + public void testEmptyCSVReader() throws IOException { + RowIter rowIter = new RowIter(0,0); + + TableDataReader tableDataReader = new TableDataReader(rowIter); + + char[] first10 = new char [10]; + char [] buf = new char[10]; + for( int i = 0 ; i < 10 ; i++ ){ + first10[i] = '\0'; + buf[i] = '\0'; + } + + tableDataReader.read(buf,0,10); + +// System.out.println("first10 : " + Arrays.toString(first10)); +// System.out.println("buf : " + Arrays.toString(buf)); + Assert.assertArrayEquals(first10,buf); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab362d06/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/XMLParserTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/XMLParserTest.java b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/XMLParserTest.java new file mode 100644 index 0000000..3c072e3 --- /dev/null +++ b/contrib/views/hive-next/src/test/java/org/apache/ambari/view/hive2/resources/upload/XMLParserTest.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive2.resources.upload; + +import org.apache.ambari.view.hive2.client.Row; +import org.apache.ambari.view.hive2.resources.uploads.parsers.xml.XMLParser; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; + +public class XMLParserTest { + + @Test(expected = IOException.class) + public void testEmptyStream() throws Exception { + String xml = ""; + + try( + StringReader sr = new StringReader(xml); + XMLParser jp = new XMLParser(sr, null); + ) { + // creation of XMLParser will throw exception. + } + } + + @Test + public void testEmptyRow() throws Exception { + String xml = "<table><row></row></table>"; + try( + StringReader sr = new StringReader(xml); + XMLParser jp = new XMLParser(sr, null); + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator should not be Empty", true, iterator.hasNext()); + Assert.assertArrayEquals("Row should be empty",new Object[]{},iterator.next().getRow()); + } + } + + + @Test + public void testEmptyTable() throws Exception { + String xml = "<table></table>"; + + try( + StringReader sr = new StringReader(xml); + XMLParser jp = new XMLParser(sr, null); + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator Empty!", false, iterator.hasNext()); + } + } + + @Test + public void testParse1Row() throws Exception { + + String xml = + "<table>" + + "<row>" + + "<col name=\"key1\">value1</col>" + + "<col name=\"key2\">c</col>" + + "<col name=\"key3\">10</col>" + + "<col name=\"key4\">10.1</col>" + + "</row>" + + "</table>" ; + + try( + StringReader sr = new StringReader(xml); + XMLParser jp = new XMLParser(sr, null) + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Iterator Empty!", true, iterator.hasNext()); + Row row = iterator.next(); + Row expected = new Row(new Object[]{"value1", "c", "10", "10.1"}); + Assert.assertEquals("Row not equal!", expected, row); + + Assert.assertEquals("Should report no more rows!", false, iterator.hasNext()); + } + } + + @Test + public void testParseMultipleRow() throws Exception { + String xml = + "<table>" + + "<row>" + + "<col name=\"key1\">value1</col>" + + "<col name=\"key2\">c</col>" + + "<col name=\"key3\">10</col>" + + "<col name=\"key4\">10.1</col>" + + "</row>" + + "<row>" + + "<col name=\"key1\">value2</col>" + + "<col name=\"key2\">c2</col>" + + "<col name=\"key3\">102</col>" + + "<col name=\"key4\">true</col>" + + "</row>" + + "</table>" ; + + try( + StringReader sr = new StringReader(xml); + XMLParser jp = new XMLParser(sr, null) + ) { + Iterator<Row> iterator = jp.iterator(); + + Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 1st row!", new Row(new Object[]{"value1", "c", "10", "10.1"}), iterator.next()); + + Assert.assertEquals("Failed to detect 2nd row!", true, iterator.hasNext()); + Assert.assertEquals("Failed to match 2nd row!", new Row(new Object[]{"value2", "c2", "102", Boolean.TRUE.toString()}), iterator.next()); + + Assert.assertEquals("Failed to detect end of rows!", false, iterator.hasNext()); + Assert.assertEquals("Failed to detect end of rows 2nd time!", false, iterator.hasNext()); + } + } +}
