This is an automated email from the ASF dual-hosted git repository. east pushed a commit to branch east_check_tsfile in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit ba9a2c96da567326005e6404e4cef373b947ce38 Author: mdf369 <[email protected]> AuthorDate: Thu Feb 28 10:14:50 2019 +0800 add TsfileChecker --- iotdb/pom.xml | 28 +++ .../bufferwrite/RestorableTsFileIOWriter.java | 8 + .../org/apache/iotdb/db/utils/TsFileChecker.java | 268 +++++++++++++++++++++ .../apache/iotdb/tsfile/TsFileSequenceRead.java | 28 +-- .../tsfile/file/metadata/ChunkGroupMetaData.java | 6 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 4 + 6 files changed, 325 insertions(+), 17 deletions(-) diff --git a/iotdb/pom.xml b/iotdb/pom.xml index 7ff37e9..b8eea29 100644 --- a/iotdb/pom.xml +++ b/iotdb/pom.xml @@ -134,6 +134,11 @@ <artifactId>maven-jar-plugin</artifactId> <configuration> <outputDirectory>${project.basedir}/iotdb/lib</outputDirectory> + <archive> + <manifest> + <mainClass>org.apache.iotdb.db.utils.TsFileChecker</mainClass> + </manifest> + </archive> </configuration> </plugin> <!--using `mvn test` to run UT, `mvn verify` to run ITs @@ -163,6 +168,29 @@ <skipITs>${iotdb.it.skip}</skipITs> </configuration> </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.4.1</version> + <configuration> + <archive> + <manifest> + <mainClass>org.apache.iotdb.db.utils.TsFileChecker</mainClass> + </manifest> + </archive> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> <!-- Might require this in Eclipse --> <!--plugin> <groupId>org.codehaus.mojo</groupId> diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java index 7331104..6e909a9 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java @@ -133,6 +133,14 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { } + public static int getTsPositionByteSize() { + return TS_POSITION_BYTE_SIZE; + } + + public static String getRestoreSuffix() { + return RESTORE_SUFFIX; + } + private void recoverMetadata(List<ChunkGroupMetaData> rowGroupMetaDatas) { // TODO it is better if we can consider the problem caused by deletion // and re-create time series here. diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/TsFileChecker.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/TsFileChecker.java new file mode 100644 index 0000000..3c78ec4 --- /dev/null +++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/TsFileChecker.java @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import org.apache.iotdb.db.engine.bufferwrite.RestorableTsFileIOWriter; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.encoding.decoder.Decoder; +import org.apache.iotdb.tsfile.file.MetaMarker; +import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; +import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.reader.page.PageReader; +import org.apache.iotdb.tsfile.utils.BytesUtils; + +public class TsFileChecker { + + public static int checkTsFile(String path) { + TsFileSequenceReader reader = null; + long lastPosition = Long.MAX_VALUE; + try { + boolean complete = true; + String restoreFilePath = path + RestorableTsFileIOWriter.getRestoreSuffix(); + long tsFileLen = new File(path).length(); + try { + lastPosition = readLastPositionFromRestoreFile(restoreFilePath); + complete = false; + } catch (IOException ex) { + lastPosition = tsFileLen; + complete = true; + } + + if (lastPosition > tsFileLen) { + return 0; + } + if (lastPosition <= TSFileConfig.MAGIC_STRING.length()) { + return 2; + } + + reader = new TsFileSequenceReader(path, complete); + if (!complete) { + reader.position(TSFileConfig.MAGIC_STRING.length()); + } + reader.readHeadMagic(); + byte marker; + while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) { + if (reader.position() > lastPosition) { + return 3; + } + + switch (marker) { + case MetaMarker.CHUNK_HEADER: + ChunkHeader header = reader.readChunkHeader(); + Decoder defaultTimeDecoder = Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().timeSeriesEncoder), + TSDataType.INT64); + Decoder valueDecoder = Decoder + .getDecoderByType(header.getEncodingType(), header.getDataType()); + for (int j = 0; j < header.getNumOfPages(); j++) { + PageHeader pageHeader = reader.readPageHeader(header.getDataType()); + ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType()); + PageReader reader1 = new PageReader(pageData, header.getDataType(), valueDecoder, + defaultTimeDecoder); + while (reader1.hasNextBatch()) { + BatchData batchData = reader1.nextBatch(); + while (batchData.hasNext()) { + batchData.currentTime(); + batchData.currentValue(); + batchData.next(); + } + } + } + break; + case MetaMarker.CHUNK_GROUP_FOOTER: + reader.readChunkGroupFooter(); + if (reader.position() == lastPosition) { + return 2; + } + break; + default: + MetaMarker.handleUnexpectedMarker(marker); + } + } + + TsFileMetaData metaData = reader.readFileMetadata(); + List<TsDeviceMetadataIndex> deviceMetadataIndexList = metaData.getDeviceMap().values().stream() + .sorted((x, y) -> (int) (x.getOffset() - y.getOffset())).collect(Collectors.toList()); + for (TsDeviceMetadataIndex index : deviceMetadataIndexList) { + TsDeviceMetadata deviceMetadata = reader.readTsDeviceMetaData(index); + deviceMetadata.getChunkGroupMetaDataList(); + } + reader.readTailMagic(); + reader.close(); + + return 1; + } catch (Exception ex) { +// System.out.println(ex); + try { + if (reader != null || reader.position() > lastPosition) { + return 3; + } else { + return 0; + } + } catch (IOException e) { + return 0; + } + } + } + + private static long readLastPositionFromRestoreFile(String path) throws IOException { + int tsfilePositionByteSize = RestorableTsFileIOWriter.getTsPositionByteSize(); + byte[] lastPostionBytes = new byte[tsfilePositionByteSize]; + RandomAccessFile randomAccessFile = null; + randomAccessFile = new RandomAccessFile(path, "r"); + + long fileLength = randomAccessFile.length(); + randomAccessFile.seek(fileLength - tsfilePositionByteSize); + randomAccessFile.read(lastPostionBytes); + long lastPosition = BytesUtils.bytesToLong(lastPostionBytes); + randomAccessFile.close(); + return lastPosition; + } + + public static List<File> getFileList(String dirPath) { + List<File> res = new ArrayList<>(); + File dir = new File(dirPath); + if (dir.isFile()) { + res.add(dir); + return res; + } + + File[] files = dir.listFiles(); + if (files != null) { + for (int i = 0; i < files.length; i++) { + if (files[i].isDirectory()) { + res.addAll(getFileList(files[i].getAbsolutePath())); + } else { + res.add(files[i]); + } + } + + } + return res; + } + + public static List<File> getLastDirList(String dirPath) { + List<File> res = new ArrayList<>(); + File dir = new File(dirPath); + if (dir.isFile()) { + return Collections.emptyList(); + } + + File[] files = dir.listFiles(); + if (files != null) { + for (int i = 0; i < files.length; i++) { + if (files[i].isDirectory()) { + res.addAll(getLastDirList(files[i].getAbsolutePath())); + } else { + res.add(dir); + return res; + } + } + + } + return res; + } + + public static int getRestoreFileNum(File dir) throws Exception { + int count = 0; + if (dir.isFile()) { + throw new Exception("Path: " + dir.getAbsolutePath() + " is a file!"); + } + + File[] files = dir.listFiles(); + if (files != null) { + for (int i = 0; i < files.length; i++) { + if (files[i].isFile() && files[i].getPath().endsWith(RestorableTsFileIOWriter.getRestoreSuffix())) { + count++; + } + } + + } + return count; + } + + public static void main(String[] args) throws Exception { +// args = new String[]{"/Users/East/Desktop/tsfile"}; + + if (args == null || args.length < 1) { + System.out.println("Please input root dir!"); + System.exit(1); + } + + String root = args[0]; + + List<File> lastDirList = TsFileChecker.getLastDirList(root); + List<String> wrongRestoreNumDirList = new ArrayList<>(); + for (int i = 0; i < lastDirList.size(); i++) { + int count = TsFileChecker.getRestoreFileNum(lastDirList.get(i)); + if (count > 1) { + wrongRestoreNumDirList.add(lastDirList.get(i).getAbsolutePath()); + } + } + System.out.println("Directories of more than one restore file:"); + for (int i = 0; i < wrongRestoreNumDirList.size(); i++) { + System.out.println(wrongRestoreNumDirList.get(i)); + } + System.out.println(); + + List<File> fileList = TsFileChecker.getFileList(root); + Map<Integer, List<String>> tsfileStatusMap = new HashMap<>(); + tsfileStatusMap.put(0, new ArrayList<>()); + tsfileStatusMap.put(1, new ArrayList<>()); + tsfileStatusMap.put(2, new ArrayList<>()); + tsfileStatusMap.put(3, new ArrayList<>()); + int num = fileList.size(); + System.out.println("Num of file: " + num); + for (int i = 0; i < fileList.size(); i++) { + System.out.println("Check No." + (i + 1)); + String filePath = fileList.get(i).getAbsolutePath(); + if (filePath.endsWith(RestorableTsFileIOWriter.getRestoreSuffix())) { + num--; + continue; + } + int status = TsFileChecker.checkTsFile(filePath); + tsfileStatusMap.get(status).add(filePath); + } + System.out.println("Num of TsFile: " + num); + System.out.println("TsFile status:"); + for (Entry<Integer, List<String>> integerListEntry : tsfileStatusMap.entrySet()) { + System.out.println(integerListEntry.getKey() + ": " + integerListEntry.getValue().size()); +// integerListEntry.getValue().forEach(value -> System.out.println(value)); + } + } +} diff --git a/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java index f5caeaa..449f953 100644 --- a/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java +++ b/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java @@ -43,7 +43,7 @@ import org.apache.iotdb.tsfile.read.reader.page.PageReader; public class TsFileSequenceRead { public static void main(String[] args) throws IOException { - TsFileSequenceReader reader = new TsFileSequenceReader("test.tsfile"); + TsFileSequenceReader reader = new TsFileSequenceReader("/Users/East/Desktop/tsfile/12272464235000-1550256931390"); System.out.println("file length: " + new File("test.tsfile").length()); System.out.println("file magic head: " + reader.readHeadMagic()); System.out.println("file magic tail: " + reader.readTailMagic()); @@ -75,19 +75,19 @@ public class TsFileSequenceRead { System.out.println("\t\tPage data position: " + reader.position()); System.out.println("\t\tpoints in the page: " + pageHeader.getNumOfValues()); ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType()); - System.out - .println("\t\tUncompressed page data size: " + pageHeader.getUncompressedSize()); - PageReader reader1 = new PageReader(pageData, header.getDataType(), valueDecoder, - defaultTimeDecoder); - while (reader1.hasNextBatch()) { - BatchData batchData = reader1.nextBatch(); - while (batchData.hasNext()) { - System.out.println( - "\t\t\ttime, value: " + batchData.currentTime() + ", " + batchData - .currentValue()); - batchData.next(); - } - } +// System.out +// .println("\t\tUncompressed page data size: " + pageHeader.getUncompressedSize()); +// PageReader reader1 = new PageReader(pageData, header.getDataType(), valueDecoder, +// defaultTimeDecoder); +// while (reader1.hasNextBatch()) { +// BatchData batchData = reader1.nextBatch(); +// while (batchData.hasNext()) { +// System.out.println( +// "\t\t\ttime, value: " + batchData.currentTime() + ", " + batchData +// .currentValue()); +// batchData.next(); +// } +// } } break; case MetaMarker.CHUNK_GROUP_FOOTER: diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java index 01cc0ab..b0a8774 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java @@ -127,9 +127,9 @@ public class ChunkGroupMetaData { ChunkGroupMetaData chunkGroupMetaData = new ChunkGroupMetaData(); chunkGroupMetaData.deviceID = ReadWriteIOUtils.readString(buffer); - chunkGroupMetaData.startOffsetOfChunkGroup = ReadWriteIOUtils.readLong(buffer); - chunkGroupMetaData.endOffsetOfChunkGroup = ReadWriteIOUtils.readLong(buffer); - chunkGroupMetaData.version = ReadWriteIOUtils.readLong(buffer); +// chunkGroupMetaData.startOffsetOfChunkGroup = ReadWriteIOUtils.readLong(buffer); +// chunkGroupMetaData.endOffsetOfChunkGroup = ReadWriteIOUtils.readLong(buffer); +// chunkGroupMetaData.version = ReadWriteIOUtils.readLong(buffer); int size = ReadWriteIOUtils.readInt(buffer); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index fa673ba..19f781d 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -267,6 +267,10 @@ public class TsFileSequenceReader { return tsFileInput.position(); } + public void position(long position) throws IOException { + tsFileInput.position(position); + } + public ByteBuffer readPage(PageHeader header, CompressionType type) throws IOException { return readPage(header, type, -1); }
