[CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation of multi reader for non-transactional table
Issue : CarbonIterator inside CarbonRecordReader was keeping reference of RowBatch and it is not being closed inside CarbonRecordReader. sort_column with measure was considering all the dimension column along with given column. if creation of one CarbonReader for non transactional table is failed then we are not able to create another CarbonReader. Solution : close() called inside hasNext() to clear the previous iterator before iterating over the next CarbonReader.. if sortcolumn is not empty and sortcolumnsList contains the fields check is added to finally in sortcolumn props. Clear the datamap in catch block, if creation of CarbonReader is failed. This closes #2362 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/19312ab5 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/19312ab5 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/19312ab5 Branch: refs/heads/carbonstore Commit: 19312ab5b35654e89f04ec79881d928c859f8a07 Parents: 290ef5a Author: rahul <rahul.ku...@knoldus.in> Authored: Tue Jun 5 13:09:36 2018 +0530 Committer: kumarvishal09 <kumarvishal1...@gmail.com> Committed: Tue Jun 12 17:01:53 2018 +0530 ---------------------------------------------------------------------- .../createTable/TestNonTransactionalCarbonTable.scala | 7 ++++--- .../java/org/apache/carbondata/sdk/file/CarbonReader.java | 2 ++ .../org/apache/carbondata/sdk/file/CarbonWriterBuilder.java | 5 ----- 3 files changed, 6 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala index b275bb8..805fc71 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala @@ -368,8 +368,9 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { |'carbondata' LOCATION |'$writerPath' """.stripMargin) - checkExistence(sql("describe formatted sdkOutputTable"), true, "age") - + checkExistence(sql("describe formatted sdkOutputTable"), true, "SORT_COLUMNS age") + checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS name,age") + checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS age,name") buildTestDataSingleFile() assert(new File(writerPath).exists()) sql("DROP TABLE IF EXISTS sdkOutputTable") @@ -402,7 +403,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { intercept[RuntimeException] { buildTestDataWithSortColumns(List("")) } - + assert(!(new File(writerPath).exists())) cleanTestData() } http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java index 9af710f..be809e6 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java @@ -74,6 +74,8 @@ public class CarbonReader<T> { return false; } else { index++; + // current reader is closed + currentReader.close(); currentReader = readers.get(index); return currentReader.nextKeyValue(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java index bd64568..0f670fe 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java @@ -484,11 +484,6 @@ public class CarbonWriterBuilder { if (isSortColumn > -1) { columnSchema.setSortColumn(true); sortColumnsSchemaList[isSortColumn] = columnSchema; - } else if (!sortColumnsList.isEmpty() && columnSchema.isDimensionColumn() - && columnSchema.getNumberOfChild() < 1) { - columnSchema.setSortColumn(true); - sortColumnsSchemaList[i] = columnSchema; - i++; } } }