Repository: carbondata Updated Branches: refs/heads/master 5d55b83c1 -> 3ea2a1dd5
[CARBONDATA-2411] infinite loop when sdk writer throws Exception Exception is not handled when Object is cast to String[] in writer method of CarbonWriter so no Exception thrown to caller and data-loading threads is not interrupted which is leading complete sdk application to infinite loop ( Or data loading max timeout 2 Day). Throw Exception to caller and one Exception call writer close method This closes #2245 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3ea2a1dd Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3ea2a1dd Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3ea2a1dd Branch: refs/heads/master Commit: 3ea2a1dd52f907fcf44966053a1a06a640654aae Parents: 5d55b83 Author: BJangir <[email protected]> Authored: Sat Apr 28 05:45:04 2018 +0530 Committer: kumarvishal09 <[email protected]> Committed: Mon Apr 30 18:09:19 2018 +0530 ---------------------------------------------------------------------- .../carbondata/sdk/file/AvroCarbonWriter.java | 13 ++++--- .../carbondata/sdk/file/CSVCarbonWriter.java | 5 ++- .../sdk/file/CarbonWriterBuilder.java | 20 ++++++---- .../sdk/file/CSVCarbonWriterTest.java | 41 ++++++++++++++++++++ 4 files changed, 64 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ea2a1dd/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java index e88164c..046a4ee 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java @@ -99,14 +99,15 @@ class AvroCarbonWriter extends CarbonWriter { */ @Override public void write(Object object) throws IOException { - GenericData.Record record = (GenericData.Record) object; - - // convert Avro record to CSV String[] - String[] csvRecord = avroToCsv(record); - writable.set(csvRecord); try { + GenericData.Record record = (GenericData.Record) object; + + // convert Avro record to CSV String[] + String[] csvRecord = avroToCsv(record); + writable.set(csvRecord); recordWriter.write(NullWritable.get(), writable); - } catch (InterruptedException e) { + } catch (Exception e) { + close(); throw new IOException(e); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ea2a1dd/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java index df6afc6..62d5860 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java @@ -65,10 +65,11 @@ class CSVCarbonWriter extends CarbonWriter { */ @Override public void write(Object object) throws IOException { - writable.set((String[]) object); try { + writable.set((String[]) object); recordWriter.write(NullWritable.get(), writable); - } catch (InterruptedException e) { + } catch (Exception e) { + close(); throw new IOException(e); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ea2a1dd/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java index 5f5ee6f..770c6b0 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java @@ -334,11 +334,15 @@ public class CarbonWriterBuilder { // If sort columns are not specified, default set all dimensions to sort column. // When dimensions are default set to sort column, // Inverted index will be supported by default for sort columns. + //Null check for field to handle hole in field[] ex. + // user passed size 4 but supplied only 2 fileds for (Field field : schema.getFields()) { - if (field.getDataType() == DataTypes.STRING || - field.getDataType() == DataTypes.DATE || - field.getDataType() == DataTypes.TIMESTAMP) { - sortColumnsList.add(field.getFieldName()); + if (null != field) { + if (field.getDataType() == DataTypes.STRING || + field.getDataType() == DataTypes.DATE || + field.getDataType() == DataTypes.TIMESTAMP) { + sortColumnsList.add(field.getFieldName()); + } } } sortColumns = new String[sortColumnsList.size()]; @@ -347,9 +351,11 @@ public class CarbonWriterBuilder { sortColumnsList = Arrays.asList(sortColumns); } for (Field field : schema.getFields()) { - tableSchemaBuilder.addColumn( - new StructField(field.getFieldName(), field.getDataType()), - sortColumnsList.contains(field.getFieldName())); + if (null != field) { + tableSchemaBuilder.addColumn( + new StructField(field.getFieldName(), field.getDataType()), + sortColumnsList.contains(field.getFieldName())); + } } String tableName; String dbName; http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ea2a1dd/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java index 41fde66..76778e5 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java @@ -205,4 +205,45 @@ public class CSVCarbonWriterTest { FileUtils.deleteDirectory(new File(path)); } + + @Test(expected = IOException.class) + public void testWhenWriterthrowsError() throws IOException{ + CarbonWriter carbonWriter = null; + String path = "./testWriteFiles"; + + FileUtils.deleteDirectory(new File(path)); + Field[] fields = new Field[2]; + fields[0] = new Field("name", DataTypes.STRING); + fields[1] = new Field("age", DataTypes.INT); + try { + carbonWriter = CarbonWriter.builder().isTransactionalTable(false). + outputPath(path).withSchema(new Schema(fields)).buildWriterForCSVInput(); + } catch (InvalidLoadOptionException e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + carbonWriter.write("babu,1"); + carbonWriter.close(); + + } + @Test + public void testWrongSchemaFieldsValidation() throws IOException{ + CarbonWriter carbonWriter = null; + String path = "./testWriteFiles"; + + FileUtils.deleteDirectory(new File(path)); + Field[] fields = new Field[3]; // supply 3 size fields but actual Field array value given is 2 + fields[0] = new Field("name", DataTypes.STRING); + fields[1] = new Field("age", DataTypes.INT); + try { + carbonWriter = CarbonWriter.builder().isTransactionalTable(false). + outputPath(path).withSchema(new Schema(fields)).buildWriterForCSVInput(); + } catch (InvalidLoadOptionException e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + carbonWriter.write(new String[]{"babu","1"}); + carbonWriter.close(); + + } }
