Repository: carbondata Updated Branches: refs/heads/master 43ffe79f3 -> 0e39abf81
[CARBONDATA-3051] Fix bugs in unclosed streams Failed to run CarbonReaderTests in windows env. This is due to two problems in carbondata: Problem 1: While encountering problems, we should close all the streams instead of only close current one. Problem 2: Current FileReader is not singleton nor the fileCache variable is a class variable, so it can only provide cache function if we pass the object among the methods. After reading version info, the fileReader cannot be used by other methods, so we must close it. This closes #2871 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0e39abf8 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0e39abf8 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0e39abf8 Branch: refs/heads/master Commit: 0e39abf81b0c7279abbecb9a4a387e447054a29a Parents: 43ffe79 Author: xuchuanyin <[email protected]> Authored: Mon Oct 29 15:33:10 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Wed Oct 31 16:27:00 2018 +0800 ---------------------------------------------------------------------- .../java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java | 3 ++- .../java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java | 1 + .../java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e39abf8/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java index 9651a8f..cc3fd72 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java @@ -29,6 +29,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.util.CarbonSessionInfo; +import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.ThreadLocalSessionInfo; import org.apache.carbondata.hadoop.api.CarbonFileInputFormat; @@ -169,7 +170,7 @@ public class CarbonReaderBuilder { reader.initialize(split, attempt); readers.add(reader); } catch (Exception e) { - reader.close(); + CarbonUtil.closeStreams(readers.toArray(new RecordReader[0])); throw e; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e39abf8/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java index 53ebc53..fe8e168 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java @@ -93,6 +93,7 @@ public class CarbonSchemaReader { FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath)); ByteBuffer buffer = fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(dataFilePath), fileSize - 8, 8); + fileReader.finish(); CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFilePath, buffer.getLong()); FileFooter3 footer = footerReader.readFooterVersion3(); if (null != footer.getExtra_info()) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e39abf8/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java index 90ea909..ed697d7 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java @@ -355,6 +355,7 @@ public class CSVCarbonWriterTest { FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFile.getPath())); ByteBuffer buffer = fileReader.readByteBuffer(FileFactory.getUpdatedFilePath( dataFile.getPath()), dataFile.getSize() - 8, 8); + fileReader.finish(); CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFile.getAbsolutePath(), buffer.getLong()); FileFooter3 footer = footerReader.readFooterVersion3();
