Repository: carbondata Updated Branches: refs/heads/master eb52c37c3 -> a437425a0
[CARBONDATA-3119] Fixed SDK Write for Complex Array Type when Array is Empty What was the issue? For SDK Write , it was going into bad record by returning null on passing empty array for Complex Type. What has been changed? Added a check for empty array. This will return an empty array. This closes #2983 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a437425a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a437425a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a437425a Branch: refs/heads/master Commit: a437425a091cfe63b6127e65370457e8e8bbdb83 Parents: eb52c37 Author: shivamasn <[email protected]> Authored: Wed Dec 12 15:09:17 2018 +0530 Committer: kunal642 <[email protected]> Committed: Tue Dec 18 10:05:49 2018 +0530 ---------------------------------------------------------------------- .../complexType/TestComplexDataType.scala | 25 +++++++++++++++ .../loading/parser/impl/ArrayParserImpl.java | 4 +++ .../carbondata/sdk/file/CarbonReaderTest.java | 32 ++++++++++++++++++++ 3 files changed, 61 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/a437425a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala index 40ff648..9cbd842 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala @@ -106,6 +106,31 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { Seq(Row(1, "abc", Row(mutable.WrappedArray.make(Array("abc", "bcd"))), "bcd"))) } + test("test Projection PushDown for Array - String type when Array is Empty") { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FAIL") + sql("drop table if exists table1") + sql("create table table1 (detail array<string>) stored by 'carbondata'") + sql("insert into table1 values('')") + checkAnswer(sql("select detail[0] from table1"), Seq(Row(""))) + sql("drop table if exists table1") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, badRecordAction) + } + + test("test Projection PushDown for Struct - Array type when Array is Empty") { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FAIL") + sql("drop table if exists table1") + sql("create table table1 (person struct<detail:array<string>,age:int>) stored by 'carbondata'") + sql("insert into table1 values ('\0011')") + checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(""))) + checkAnswer(sql("select person.age from table1"), Seq(Row(1))) + sql("drop table if exists table1") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, badRecordAction) + } + test("test Projection PushDown for Struct - Double type") { sql("DROP TABLE IF EXISTS table1") sql( http://git-wip-us.apache.org/repos/asf/carbondata/blob/a437425a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java index c27f0fa..3f4fc4a 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java @@ -56,6 +56,10 @@ public class ArrayParserImpl implements ComplexParser<ArrayObject> { } return new ArrayObject(array); } + } else if (value.isEmpty()) { + Object[] array = new Object[1]; + array[0] = child.parse(value); + return new ArrayObject(array); } } return null; http://git-wip-us.apache.org/repos/asf/carbondata/blob/a437425a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index 4e11233..a3624b0 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -2072,4 +2072,36 @@ public class CarbonReaderTest extends TestCase { } } + @Test + public void testSdkWriteWhenArrayOfStringIsEmpty() throws IOException, InvalidLoadOptionException { + String badRecordAction = + CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION); + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FAIL"); + + String path = "./testSdkWriteWhenArrayOfStringIsEmpty"; + String[] rec = { "aaa", "bbb", "[email protected]", "", "", "mmm", "" }; + Field[] fields = new Field[7]; + fields[0] = new Field("stringField", DataTypes.STRING); + fields[1] = new Field("varcharField", DataTypes.VARCHAR); + fields[2] = new Field("stringField1", DataTypes.STRING); + fields[3] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING)); + fields[4] = new Field("arrayField1", DataTypes.createArrayType(DataTypes.STRING)); + fields[5] = new Field("arrayField2", DataTypes.createArrayType(DataTypes.STRING)); + fields[6] = new Field("varcharField1", DataTypes.VARCHAR); + Schema schema = new Schema(fields); + Map map = new HashMap(); + map.put("complex_delimiter_level_1", "#"); + map.put("bad_records_logger_enable", "TRUE"); + map.put("bad_record_path", path + "/badrec"); + CarbonWriterBuilder builder = CarbonWriter.builder().outputPath(path); + builder.withLoadOptions(map).withCsvInput(schema).enableLocalDictionary(false) + .writtenBy("CarbonReaderTest"); + CarbonWriter writer = builder.build(); + writer.write(rec); + writer.close(); + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, badRecordAction); + } + }
