This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch iotdb39-autoreadrepair
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/iotdb39-autoreadrepair by this
push:
new 3a7c3de fix stream unclosed when exception occurs when creating
TsFileReader
3a7c3de is described below
commit 3a7c3de411269c125a260d1ed8b5d1eef0c89f7a
Author: xiangdong huang <[email protected]>
AuthorDate: Sat Mar 16 22:56:24 2019 +0800
fix stream unclosed when exception occurs when creating TsFileReader
---
.../iotdb/tsfile/read/TsFileRestorableReader.java | 7 ++-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 9 ++-
.../tsfile/read/TsFileRestorableReaderTest.java | 1 -
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 66 ++++++++--------------
4 files changed, 38 insertions(+), 45 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
index ccbce14..44b29a6 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
@@ -43,7 +43,12 @@ public class TsFileRestorableReader extends
TsFileSequenceReader {
// Exception when reading broken data. Therefore, we set loadMetadata as
true in this case.
super(file, !autoRepair);
if (autoRepair) {
- checkAndRepair();
+ try {
+ checkAndRepair();
+ } catch (Throwable e) {
+ close();
+ throw e;
+ }
loadMetadataSize();
}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index c15ba7b..6a9ad7b 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -83,8 +83,13 @@ public class TsFileSequenceReader implements AutoCloseable{
this.file = file;
final Path path = Paths.get(file);
tsFileInput = new DefaultTsFileInput(path);
- if (loadMetadataSize) {
- loadMetadataSize();
+ try {
+ if (loadMetadataSize) {
+ loadMetadataSize();
+ }
+ } catch (Throwable e) {
+ tsFileInput.close();
+ throw e;
}
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileRestorableReaderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileRestorableReaderTest.java
index 5454c44..14b6ca3 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileRestorableReaderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileRestorableReaderTest.java
@@ -53,7 +53,6 @@ public class TsFileRestorableReaderTest {
File file = new File(FILE_PATH);
IncompleteFileTestUtil.writeFileWithOneIncompleteChunkHeader(file);
-
// This should throw an Illegal Argument Exception
TsFileSequenceReader reader = new TsFileRestorableReader(FILE_PATH, false);
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
index 064f1cc..7c01c1f 100755
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
@@ -159,10 +159,33 @@ public class TsFileGeneratorForTest {
innerWriter = new TsFileWriter(file, schema,
TSFileDescriptor.getInstance().getConfig());
// write
- try {
- writeToFile(schema);
+ try (Scanner in = new Scanner(new File(inputDataFile))) {
+ long lineCount = 0;
+ long startTime = System.currentTimeMillis();
+ long endTime = System.currentTimeMillis();
+ assert in != null;
+ while (in.hasNextLine()) {
+ if (lineCount % 1000000 == 0) {
+ endTime = System.currentTimeMillis();
+ // logger.info("write line:{},inner space consumer:{},use
+ //
time:{}",lineCount,innerWriter.calculateMemSizeForEachGroup(),endTime);
+ LOG.info("write line:{},use time:{}s", lineCount, (endTime -
startTime) / 1000);
+ }
+ String str = in.nextLine();
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema);
+ innerWriter.write(record);
+ lineCount++;
+ }
+ endTime = System.currentTimeMillis();
+ LOG.info("write line:{},use time:{}s", lineCount, (endTime - startTime)
/ 1000);
+ endTime = System.currentTimeMillis();
+ LOG.info("write total:{},use time:{}s", lineCount, (endTime - startTime)
/ 1000);
+ LOG.info("src file size:{}GB", FileUtils.getLocalFileByte(inputDataFile,
FileUtils.Unit.GB));
+ LOG.info("out file size:{}MB",
FileUtils.getLocalFileByte(outputDataFile, FileUtils.Unit.MB));
} catch (WriteProcessException e) {
e.printStackTrace();
+ } finally {
+ innerWriter.close();
}
LOG.info("write to file successfully!!");
}
@@ -228,43 +251,4 @@ public class TsFileGeneratorForTest {
return schemaBuilder.build();
}
- static public void writeToFile(FileSchema schema)
- throws InterruptedException, IOException, WriteProcessException {
- Scanner in = getDataFile(inputDataFile);
- long lineCount = 0;
- long startTime = System.currentTimeMillis();
- long endTime = System.currentTimeMillis();
- assert in != null;
- while (in.hasNextLine()) {
- if (lineCount % 1000000 == 0) {
- endTime = System.currentTimeMillis();
- // logger.info("write line:{},inner space consumer:{},use
- //
time:{}",lineCount,innerWriter.calculateMemSizeForEachGroup(),endTime);
- LOG.info("write line:{},use time:{}s", lineCount, (endTime -
startTime) / 1000);
- }
- String str = in.nextLine();
- TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema);
- innerWriter.write(record);
- lineCount++;
- }
- endTime = System.currentTimeMillis();
- LOG.info("write line:{},use time:{}s", lineCount, (endTime - startTime) /
1000);
- innerWriter.close();
- in.close();
- endTime = System.currentTimeMillis();
- LOG.info("write total:{},use time:{}s", lineCount, (endTime - startTime) /
1000);
- LOG.info("src file size:{}GB", FileUtils.getLocalFileByte(inputDataFile,
FileUtils.Unit.GB));
- LOG.info("out file size:{}MB", FileUtils.getLocalFileByte(outputDataFile,
FileUtils.Unit.MB));
- }
-
- static private Scanner getDataFile(String path) {
- File file = new File(path);
- try {
- Scanner in = new Scanner(file);
- return in;
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- return null;
- }
- }
}
