This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch iotdb39-autoreadrepair
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/iotdb39-autoreadrepair by this
push:
new 0b2aa12 add append option in TsFile writer and add a subclass of
TsFileSequenceReader for supporting auto repair
0b2aa12 is described below
commit 0b2aa125f2faa5c85e47bf814f4c676abd9b6d01
Author: xiangdong huang <[email protected]>
AuthorDate: Sat Mar 16 16:49:58 2019 +0800
add append option in TsFile writer and add a subclass of
TsFileSequenceReader for supporting auto repair
---
.../iotdb/tsfile/read/TsFileRestorableReader.java | 2 -
.../iotdb/tsfile/read/TsFileSequenceReader.java | 43 +++++++++++-----------
.../write/writer/NativeRestorableIOWriter.java | 38 ++++++++++---------
.../write/writer/NativeRestorableIOWriterTest.java | 4 ++
4 files changed, 46 insertions(+), 41 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
index 059fcf5..1c3a69f 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
@@ -52,8 +52,6 @@ public class TsFileRestorableReader extends
TsFileSequenceReader {
* Checks if the file is incomplete, and if so, tries to repair it.
*/
private void checkAndRepair(String file) throws IOException {
- long position = selfCheck(new ArrayList<>());
-
// Check if file is damaged
if (!TSFileConfig.MAGIC_STRING.equals(readTailMagic())) {
// Try to close it
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 44b8dd8..debd5fe 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -30,7 +30,6 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.OptionalLong;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.compress.IUnCompressor;
import org.apache.iotdb.tsfile.file.MetaMarker;
@@ -48,9 +47,7 @@ import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.reader.DefaultTsFileInput;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.writer.NativeRestorableIOWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -135,9 +132,9 @@ public class TsFileSequenceReader implements AutoCloseable{
// CHeck if the file is large enough to contain a tail magic
// If the file only contains a header magic, this could also be assumed to
be the tail magic
- if (totalSize <= TSFileConfig.MAGIC_STRING.length()) {
- throw new IOException("This file has no tail magic!");
- }
+// if (totalSize <= TSFileConfig.MAGIC_STRING.length()) {
+// throw new IOException("This file has no tail magic!");
+// }
ByteBuffer magicStringBytes =
ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length());
tsFileInput.read(magicStringBytes, totalSize -
TSFileConfig.MAGIC_STRING.length());
@@ -146,6 +143,16 @@ public class TsFileSequenceReader implements AutoCloseable{
}
/**
+ * whether the file is a complete TsFile: only if the head magic and tail
magic string exists.
+ * @return
+ * @throws IOException
+ */
+ public boolean isComplete() throws IOException {
+ return tsFileInput.size() == TSFileConfig.MAGIC_STRING.length() * 2 &&
readTailMagic()
+ .equals(readHeadMagic());
+ }
+
+ /**
* this function does not modify the position of the file reader.
*/
public String readHeadMagic() throws IOException {
@@ -426,27 +433,17 @@ public class TsFileSequenceReader implements
AutoCloseable{
/**
* Self Check the file and return the position before where the data is safe.
*
- * @param newMetaData @OUT can not be null, the chunk group metadta in the
file will be added into
- * this parameter. If the file is complete, then this parameter will be not
modified.
- * @return the position of the file that is fine. All data after the
position in the file should
- * be truncated.
- */
- public long selfCheck(List<ChunkGroupMetaData> newMetaData) throws
IOException {
- return selfCheck(null, newMetaData);
- }
-
- /**
- * Self Check the file and return the position before where the data is safe.
- *
* @param newSchema @OUT. the measurement schema in the file will be added
into
- * this parameter. If the file is complete, then this parameter will be not
modified.
+ * this parameter.
* @param newMetaData @OUT can not be null, the chunk group metadta in the
file will be added into
- * this parameter. If the file is complete, then this parameter will be not
modified.
+ * this parameter.
+ * @param fastFinish if true and the file is complete, then newSchema and
newMetaData parameter
+ * will be not modified.
* @return the position of the file that is fine. All data after the
position in the file should
* be truncated.
*/
public long selfCheck(Map<String, MeasurementSchema> newSchema,
- List<ChunkGroupMetaData> newMetaData) throws IOException {
+ List<ChunkGroupMetaData> newMetaData, boolean fastFinish) throws
IOException {
File checkFile = new File(this.file);
long fileSize;
if (!checkFile.exists()) {
@@ -483,7 +480,9 @@ public class TsFileSequenceReader implements AutoCloseable{
return TsFileCheckStatus.ONLY_MAGIC_HEAD;
} else if (readTailMagic().equals(magic)) {
loadMetadataSize();
- return TsFileCheckStatus.COMPLETE_FILE;
+ if (fastFinish) {
+ return TsFileCheckStatus.COMPLETE_FILE;
+ }
}
// not a complete file, we will recover it...
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriter.java
index b960e85..2496d42 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriter.java
@@ -57,27 +57,31 @@ public class NativeRestorableIOWriter extends
TsFileIOWriter {
public NativeRestorableIOWriter(File file, boolean append) throws
IOException {
super();
this.out = new DefaultTsFileOutput(file, true);
-
- try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getAbsolutePath(), false)){
- truncatedPosition = reader.selfCheck(knownSchemas,
chunkGroupMetaDataList);
- if (truncatedPosition == TsFileCheckStatus.FILE_NOT_FOUND) {
- //it is ok.. because this is a new file.
- } else if (truncatedPosition == TsFileCheckStatus.COMPLETE_FILE) {
- if (!append) {
- this.canWrite = false;
+ if (file.length() == 0) {
+ //this is a new file
+ return;
+ }
+ if (file.exists()) {
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getAbsolutePath(), false)) {
+ if (reader.isComplete() && !append) {
+ canWrite = false;
+ out.close();
+ return;
+ }
+ truncatedPosition = reader.selfCheck(knownSchemas,
chunkGroupMetaDataList, !append);
+ if (truncatedPosition == TsFileCheckStatus.COMPLETE_FILE && !append) {
+ this.canWrite = false;
+ out.close();
+ } else if (truncatedPosition == TsFileCheckStatus.INCOMPATIBLE_FILE) {
out.close();
+ throw new IOException(
+ String.format("%s is not in TsFile format.",
file.getAbsolutePath()));
+ } else if (truncatedPosition == TsFileCheckStatus.ONLY_MAGIC_HEAD) {
+ out.truncate(TSFileConfig.MAGIC_STRING.length());
} else {
- truncatedPosition = reader.getPositionOfFirstDeviceMetaIndex();
+ //remove broken data
out.truncate(truncatedPosition);
}
- } else if (truncatedPosition == TsFileCheckStatus.ONLY_MAGIC_HEAD) {
- out.truncate(TSFileConfig.MAGIC_STRING.length());
- } else if (truncatedPosition == TsFileCheckStatus.INCOMPATIBLE_FILE) {
- out.close();
- throw new IOException(String.format("%s is not in TsFile format.",
file.getAbsolutePath()));
- } else {
- //remove broken data
- out.truncate(truncatedPosition);
}
}
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriterTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriterTest.java
index 955a557..6ad7e1f 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriterTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriterTest.java
@@ -306,6 +306,10 @@ public class NativeRestorableIOWriterTest {
writer.write(new TSRecord(4, "d1").addTuple(new FloatDataPoint("s1", 5))
.addTuple(new FloatDataPoint("s2", 4)));
writer.close();
+ TsFileSequenceReader reader = new TsFileSequenceReader(FILE_NAME);
+ TsDeviceMetadataIndex index =
reader.readFileMetadata().getDeviceMap().get("d1");
+ assertEquals(2,
reader.readTsDeviceMetaData(index).getChunkGroupMetaDataList().size());
+ reader.close();
assertTrue(file.delete());
}
}
\ No newline at end of file
