[CARBONDATA-2467] sdk writer log shouldnot print null issue :
Logger prints the threadname along with log. In case of sdk writer we are assigning null for dbname and table name and table name is used to assign the thread name in CarbonTableOutputFormat and DataWriterProcessStepImpl. which leads to add null in many logs. Comparision of two array type was being wrong. Solution : assign temp value with current time in millisec to tablename instead of null. ArrayType class donot contains equls method. Hence it goes to Object.equals() which will perform == operation to compare both the object which is wrong . so add equals method to ArrayType class . This closes #2292 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/78efc7f7 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/78efc7f7 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/78efc7f7 Branch: refs/heads/branch-1.4 Commit: 78efc7f7110e152b8f3d2ec833e13278356a6a2d Parents: 2018048 Author: rahulforallp <[email protected]> Authored: Thu May 10 14:37:44 2018 +0530 Committer: ravipesala <[email protected]> Committed: Fri May 18 14:22:44 2018 +0530 ---------------------------------------------------------------------- .../core/metadata/datatype/ArrayType.java | 13 +++ .../TestNonTransactionalCarbonTable.scala | 84 ++++++++++++++++++++ .../sdk/file/CarbonWriterBuilder.java | 4 +- 3 files changed, 99 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java index c327d7f..0f0f120 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java @@ -34,4 +34,17 @@ public class ArrayType extends DataType { public DataType getElementType() { return elementType; } + + @Override public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ArrayType)) return false; + + ArrayType arrayType = (ArrayType) o; + + return elementType.equals(arrayType.elementType); + } + + @Override public int hashCode() { + return elementType.hashCode(); + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala index cc3cbb5..a15d0c2 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala @@ -1331,6 +1331,90 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { buildAvroTestDataStructOfArray(3, null) } + // ArrayOfStruct test + def buildAvroTestDataArrayOfStructWithNoSortCol(rows: Int, options: util.Map[String, String]): Any = { + FileUtils.deleteDirectory(new File(writerPath)) + + val mySchema = """ { + | "name": "address", + | "type": "record", + | "fields": [ + | { + | "name": "exp", + | "type": "int" + | }, + | { + | "name": "age", + | "type": "int" + | }, + | { + | "name": "doorNum", + | "type": { + | "type": "array", + | "items": { + | "type": "record", + | "name": "my_address", + | "fields": [ + | { + | "name": "street", + | "type": "string" + | }, + | { + | "name": "city", + | "type": "string" + | } + | ] + | } + | } + | } + | ] + |} """.stripMargin + val json = + """ {"exp":5,"age":10,"doorNum" : + |[{"street":"abc","city":"city1"}, + |{"street":"def","city":"city2"}, + |{"street":"ghi","city":"city3"}, + |{"street":"jkl","city":"city4"}]} """.stripMargin + + val fields = new Array[Field](3) + fields(0) = new Field("exp", DataTypes.INT) + fields(1) = new Field("age", DataTypes.INT) + + val fld = new util.ArrayList[StructField] + fld.add(new StructField("street", DataTypes.STRING)) + fld.add(new StructField("city", DataTypes.STRING)) + + val fld2 = new util.ArrayList[StructField] + fld2.add(new StructField("my_address", DataTypes.createStructType(fld), fld)) + fields(2) = new Field("doorNum", DataTypes.createArrayType(fld2.get(0).getDataType), fld2) + + WriteFilesWithAvroWriter(rows, mySchema, json, fields) + } + + test("Read sdk writer Avro output Record Type with no sort columns") { + buildAvroTestDataArrayOfStructWithNoSortCol(3,null) + assert(new File(writerPath).exists()) + sql("DROP TABLE IF EXISTS sdkOutputTable") + sql( + s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION + |'$writerPath' """.stripMargin) + + sql("desc formatted sdkOutputTable").show(false) + sql("select * from sdkOutputTable").show(false) + + /* + +---+---+----------------------------------------------------+ + |exp|age|doorNum | + +---+---+----------------------------------------------------+ + |5 |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]| + |5 |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]| + |5 |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]| + +---+---+----------------------------------------------------+ + */ + sql("DROP TABLE sdkOutputTable") + // drop table should not delete the files + assert(new File(writerPath).listFiles().length > 0) + } test("Read sdk writer Avro output Record Type") { buildAvroTestDataStructType() http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java index 36be65f..0f9c9d7 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java @@ -414,8 +414,8 @@ public class CarbonWriterBuilder { tableName = "_tempTable"; dbName = "_tempDB"; } else { - dbName = null; - tableName = null; + dbName = ""; + tableName = "_tempTable_" + String.valueOf(UUID); } TableSchema schema = tableSchemaBuilder.build(); schema.setTableName(tableName);
