This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new bb1284f [IOTDB-599] fix TsFileSketchTool (#1168)
bb1284f is described below
commit bb1284f7ab0ce942e0cb1bdb8c2d1f63d8428f01
Author: Haonan <[email protected]>
AuthorDate: Fri May 8 20:08:17 2020 +0800
[IOTDB-599] fix TsFileSketchTool (#1168)
* fix TsFileSketchTool
---
docs/SystemDesign/1-TsFile/2-Format.md | 283 ++++++++++++++++-----
.../apache/iotdb/db/tools/TsFileSketchTool.java | 104 +++++---
.../writelog/recover/TsFileRecoverPerformer.java | 2 +-
.../recover/RecoverResourceFromReaderTest.java | 189 ++++++++++++++
.../file/metadata/statistics/BinaryStatistics.java | 2 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 16 +-
6 files changed, 483 insertions(+), 113 deletions(-)
diff --git a/docs/SystemDesign/1-TsFile/2-Format.md
b/docs/SystemDesign/1-TsFile/2-Format.md
index 2c00ec6..a2aac9c 100644
--- a/docs/SystemDesign/1-TsFile/2-Format.md
+++ b/docs/SystemDesign/1-TsFile/2-Format.md
@@ -390,83 +390,230 @@ For Linux or MacOs:
- Note that if `<path of the file for saving the output result>` is not set,
the default path "TsFile_sketch_view.txt" will be used.
-An example on Windows:
+An example on macOS:
```$xslt
-D:\incubator-iotdb\server\target\iotdb-server-0.10.0\tools\tsfileToolSet>.\print-tsfile-sketch.bat
D:\data\data\sequence\root.vehicle\1572496142067-101-0.tsfile
+/incubator-iotdb/server/target/iotdb-server-0.10.0/tools/tsfileToolSet$
./print-tsfile-sketch.sh test.tsfile
````````````````````````
Starting Printing the TsFile Sketch
````````````````````````
-TsFile path:D:\data\data\sequence\root.vehicle\1572496142067-101-0.tsfile
+TsFile path:test.tsfile
Sketch save path:TsFile_sketch_view.txt
-------------------------------- TsFile Sketch --------------------------------
-file path: D:\data\data\sequence\root.vehicle\1572496142067-101-0.tsfile
-file length: 187382
-
- POSITION| CONTENT
- -------- -------
- 0| [magic head] TsFile
- 6| [version number] 000001
-||||||||||||||||||||| [Chunk Group] of root.vehicle.d0 begins at pos 12,
ends at pos 186469, version:102, num of Chunks:6
- 12| [Chunk] of s3, numOfPoints:10600, time
range:[3000,13599], tsDataType:TEXT,
-
TsDigest:[min_value:A,max_value:E,first_value:A,last_value:E,sum_value:0.0]
- | [marker] 1
- | [ChunkHeader]
- | 11 pages
- 55718| [Chunk] of s4, numOfPoints:10600, time
range:[3000,13599], tsDataType:BOOLEAN,
-
TsDigest:[min_value:false,max_value:true,first_value:true,last_value:false,sum_value:0.0]
- | [marker] 1
- | [ChunkHeader]
- | 11 pages
- 68848| [Chunk] of s5, numOfPoints:10600, time
range:[3000,13599], tsDataType:DOUBLE,
-
TsDigest:[min_value:3000.0,max_value:13599.0,first_value:3000.0,last_value:13599.0,sum_value:8.79747E7]
- | [marker] 1
- | [ChunkHeader]
- | 11 pages
- 98474| [Chunk] of s0, numOfPoints:21900, time
range:[3000,100999], tsDataType:INT32,
-
TsDigest:[min_value:0,max_value:99,first_value:0,last_value:19,sum_value:889750.0]
- | [marker] 1
- | [ChunkHeader]
- | 22 pages
- 123369| [Chunk] of s1, numOfPoints:21900, time
range:[3000,100999], tsDataType:INT64,
-
TsDigest:[min_value:0,max_value:39,first_value:8,last_value:19,sum_value:300386.0]
- | [marker] 1
- | [ChunkHeader]
- | 22 pages
- 144741| [Chunk] of s2, numOfPoints:21900, time
range:[3000,100999], tsDataType:FLOAT,
-
TsDigest:[min_value:0.0,max_value:122.0,first_value:8.0,last_value:52.0,sum_value:778581.0]
- | [marker] 1
- | [ChunkHeader]
- | 22 pages
- 186437| [Chunk Group Footer]
- | [marker] 0
- | [deviceID] root.vehicle.d0
- | [dataSize] 186425
- | [num of chunks] 6
-||||||||||||||||||||| [Chunk Group] of root.vehicle.d0 ends
- 186469| [marker] 2
- 186470| [TsDeviceMetadata] of root.vehicle.d0, startTime:3000,
endTime:100999
- | [startTime] 3000tfi
- | [endTime] 100999
- | [num of ChunkGroupMetaData] 1
- | 1 ChunkGroupMetaData
- 187133| [TsFileMetaData]
- | [num of devices] 1
- | 1 key&TsDeviceMetadataIndex
- | [num of measurements] 6
- | 6 key&measurementSchema
- | [createBy isNotNull] false
- | [totalChunkNum] 6
- | [invalidChunkNum] 0
- | [bloom filter bit vector byte array length] 31
- | [bloom filter bit vector byte array]
- | [bloom filter number of bits] 256
- | [bloom filter number of hash functions] 5
- 187372| [TsFileMetaDataSize] 239
- 187376| [magic tail] TsFile
- 187382| END of TsFile
+file path: test.tsfile
+file length: 33436
+
+ POSITION| CONTENT
+ -------- -------
+ 0| [magic head] TsFile
+ 6| [version number] 000002
+||||||||||||||||||||| [Chunk Group] of root.group_12.d2, num of Chunks:3
+ 12| [Chunk] of s_INT64e_RLE, numOfPoints:10000, time
range:[1,10000], tsDataType:INT64,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]
+ | [marker] 1
+ | [ChunkHeader]
+ | 2 pages
+ 677| [Chunk] of s_INT64e_TS_2DIFF, numOfPoints:10000, time
range:[1,10000], tsDataType:INT64,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 1349| [Chunk] of s_INT64e_PLAIN, numOfPoints:10000, time
range:[1,10000], tsDataType:INT64,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]
+ | [marker] 1
+ | [ChunkHeader]
+ | 2 pages
+ 5766| [Chunk Group Footer]
+ | [marker] 0
+ | [deviceID] root.group_12.d2
+ | [dataSize] 5754
+ | [num of chunks] 3
+||||||||||||||||||||| [Chunk Group] of root.group_12.d2 ends
+ 5799| [Version Info pair]
+ | [marker] 3
+ | [offset] 5808
+ | [version] 102
+||||||||||||||||||||| [Chunk Group] of root.group_12.d1, num of Chunks:3
+ 5808| [Chunk] of s_INT32e_PLAIN, numOfPoints:10000, time
range:[1,10000], tsDataType:INT32,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 8231| [Chunk] of s_INT32e_TS_2DIFF, numOfPoints:10000, time
range:[1,10000], tsDataType:INT32,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 8852| [Chunk] of s_INT32e_RLE, numOfPoints:10000, time
range:[1,10000], tsDataType:INT32,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 9399| [Chunk Group Footer]
+ | [marker] 0
+ | [deviceID] root.group_12.d1
+ | [dataSize] 3591
+ | [num of chunks] 3
+||||||||||||||||||||| [Chunk Group] of root.group_12.d1 ends
+ 9432| [Version Info pair]
+ | [marker] 3
+ | [offset] 9441
+ | [version] 102
+||||||||||||||||||||| [Chunk Group] of root.group_12.d0, num of Chunks:2
+ 9441| [Chunk] of s_BOOLEANe_RLE, numOfPoints:10000, time
range:[1,10000], tsDataType:BOOLEAN,
+ startTime: 1 endTime: 10000 count: 10000
[firstValue:true,lastValue:true]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 9968| [Chunk] of s_BOOLEANe_PLAIN, numOfPoints:10000, time
range:[1,10000], tsDataType:BOOLEAN,
+ startTime: 1 endTime: 10000 count: 10000
[firstValue:true,lastValue:true]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 10961| [Chunk Group Footer]
+ | [marker] 0
+ | [deviceID] root.group_12.d0
+ | [dataSize] 1520
+ | [num of chunks] 2
+||||||||||||||||||||| [Chunk Group] of root.group_12.d0 ends
+ 10994| [Version Info pair]
+ | [marker] 3
+ | [offset] 11003
+ | [version] 102
+||||||||||||||||||||| [Chunk Group] of root.group_12.d5, num of Chunks:1
+ 11003| [Chunk] of s_TEXTe_PLAIN, numOfPoints:10000, time
range:[1,10000], tsDataType:TEXT,
+ startTime: 1 endTime: 10000 count: 10000
[firstValue:version_test,lastValue:version_test]
+ | [marker] 1
+ | [ChunkHeader]
+ | 3 pages
+ 19278| [Chunk Group Footer]
+ | [marker] 0
+ | [deviceID] root.group_12.d5
+ | [dataSize] 8275
+ | [num of chunks] 1
+||||||||||||||||||||| [Chunk Group] of root.group_12.d5 ends
+ 19311| [Version Info pair]
+ | [marker] 3
+ | [offset] 19320
+ | [version] 102
+||||||||||||||||||||| [Chunk Group] of root.group_12.d4, num of Chunks:4
+ 19320| [Chunk] of s_DOUBLEe_PLAIN, numOfPoints:10000, time
range:[1,10000], tsDataType:DOUBLE,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00000000123]
+ | [marker] 1
+ | [ChunkHeader]
+ | 2 pages
+ 23740| [Chunk] of s_DOUBLEe_TS_2DIFF, numOfPoints:10000, time
range:[1,10000], tsDataType:DOUBLE,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.000000002045]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 24414| [Chunk] of s_DOUBLEe_GORILLA, numOfPoints:10000, time
range:[1,10000], tsDataType:DOUBLE,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.000000002045]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 25054| [Chunk] of s_DOUBLEe_RLE, numOfPoints:10000, time
range:[1,10000], tsDataType:DOUBLE,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.000000001224]
+ | [marker] 1
+ | [ChunkHeader]
+ | 2 pages
+ 25717| [Chunk Group Footer]
+ | [marker] 0
+ | [deviceID] root.group_12.d4
+ | [dataSize] 6397
+ | [num of chunks] 4
+||||||||||||||||||||| [Chunk Group] of root.group_12.d4 ends
+ 25750| [Version Info pair]
+ | [marker] 3
+ | [offset] 25759
+ | [version] 102
+||||||||||||||||||||| [Chunk Group] of root.group_12.d3, num of Chunks:4
+ 25759| [Chunk] of s_FLOATe_GORILLA, numOfPoints:10000, time
range:[1,10000], tsDataType:FLOAT,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00023841858]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 26375| [Chunk] of s_FLOATe_PLAIN, numOfPoints:10000, time
range:[1,10000], tsDataType:FLOAT,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00023841858]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 28796| [Chunk] of s_FLOATe_RLE, numOfPoints:10000, time
range:[1,10000], tsDataType:FLOAT,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00023841858]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 29343| [Chunk] of s_FLOATe_TS_2DIFF, numOfPoints:10000, time
range:[1,10000], tsDataType:FLOAT,
+ startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00023841858]
+ | [marker] 1
+ | [ChunkHeader]
+ | 1 pages
+ 29967| [Chunk Group Footer]
+ | [marker] 0
+ | [deviceID] root.group_12.d3
+ | [dataSize] 4208
+ | [num of chunks] 4
+||||||||||||||||||||| [Chunk Group] of root.group_12.d3 ends
+ 30000| [Version Info pair]
+ | [marker] 3
+ | [offset] 30009
+ | [version] 102
+ 30009| [marker] 2
+ 30010| [ChunkMetadataList] of
root.group_12.d0.s_BOOLEANe_PLAIN, tsDataType:BOOLEAN
+ | [startTime: 1 endTime: 10000 count: 10000
[firstValue:true,lastValue:true]]
+ 30066| [ChunkMetadataList] of root.group_12.d0.s_BOOLEANe_RLE,
tsDataType:BOOLEAN
+ | [startTime: 1 endTime: 10000 count: 10000
[firstValue:true,lastValue:true]]
+ 30120| [ChunkMetadataList] of root.group_12.d1.s_INT32e_PLAIN,
tsDataType:INT32
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]]
+ 30196| [ChunkMetadataList] of root.group_12.d1.s_INT32e_RLE,
tsDataType:INT32
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]]
+ 30270| [ChunkMetadataList] of
root.group_12.d1.s_INT32e_TS_2DIFF, tsDataType:INT32
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]]
+ 30349| [ChunkMetadataList] of root.group_12.d2.s_INT64e_PLAIN,
tsDataType:INT64
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]]
+ 30441| [ChunkMetadataList] of root.group_12.d2.s_INT64e_RLE,
tsDataType:INT64
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]]
+ 30531| [ChunkMetadataList] of
root.group_12.d2.s_INT64e_TS_2DIFF, tsDataType:INT64
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1,maxValue:1,firstValue:1,lastValue:1,sumValue:10000.0]]
+ 30626| [ChunkMetadataList] of
root.group_12.d3.s_FLOATe_GORILLA, tsDataType:FLOAT
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00023841858]]
+ 30704| [ChunkMetadataList] of root.group_12.d3.s_FLOATe_PLAIN,
tsDataType:FLOAT
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00023841858]]
+ 30780| [ChunkMetadataList] of root.group_12.d3.s_FLOATe_RLE,
tsDataType:FLOAT
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00023841858]]
+ 30854| [ChunkMetadataList] of
root.group_12.d3.s_FLOATe_TS_2DIFF, tsDataType:FLOAT
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00023841858]]
+ 30933| [ChunkMetadataList] of
root.group_12.d4.s_DOUBLEe_GORILLA, tsDataType:DOUBLE
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.000000002045]]
+ 31028| [ChunkMetadataList] of
root.group_12.d4.s_DOUBLEe_PLAIN, tsDataType:DOUBLE
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.00000000123]]
+ 31121| [ChunkMetadataList] of root.group_12.d4.s_DOUBLEe_RLE,
tsDataType:DOUBLE
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.000000001224]]
+ 31212| [ChunkMetadataList] of
root.group_12.d4.s_DOUBLEe_TS_2DIFF, tsDataType:DOUBLE
+ | [startTime: 1 endTime: 10000 count: 10000
[minValue:1.1,maxValue:1.1,firstValue:1.1,lastValue:1.1,sumValue:11000.000000002045]]
+ 31308| [ChunkMetadataList] of root.group_12.d5.s_TEXTe_PLAIN,
tsDataType:TEXT
+ | [startTime: 1 endTime: 10000 count: 10000
[firstValue:version_test,lastValue:version_test]]
+ 32840| [MetadataIndex] of root.group_12.d0
+ 32881| [MetadataIndex] of root.group_12.d1
+ 32920| [MetadataIndex] of root.group_12.d2
+ 32959| [MetadataIndex] of root.group_12.d3
+ 33000| [MetadataIndex] of root.group_12.d4
+ 33042| [MetadataIndex] of root.group_12.d5
+ 33080| [TsFileMetadata]
+ | [num of devices] 6
+ | 6 key&TsMetadataIndex
+ | [totalChunkNum] 17
+ | [invalidChunkNum] 0
+ | [bloom filter bit vector byte array length] 32
+ | [bloom filter bit vector byte array]
+ | [bloom filter number of bits] 256
+ | [bloom filter number of hash functions] 5
+ 33426| [TsFileMetadataSize] 346
+ 33430| [magic tail] TsFile
+ 33436| END of TsFile
---------------------------------- TsFile Sketch End
----------------------------------
+
````````````````````````
#### 1.3.4 TsFileSequenceRead
diff --git
a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index ba5651a..840fa73 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -22,19 +22,24 @@ package org.apache.iotdb.db.tools;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
-import java.util.LinkedHashMap;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
+import java.util.TreeMap;
+
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.MetadataIndexEntry;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
+import org.apache.iotdb.tsfile.utils.Pair;
public class TsFileSketchTool {
@@ -59,13 +64,9 @@ public class TsFileSketchTool {
// get metadata information
TsFileSequenceReader reader = new TsFileSequenceReader(filename);
TsFileMetadata tsFileMetaData = reader.readFileMetadata();
- List<String> devices = reader.getAllDevices();
- Map<String, Map<String, List<ChunkMetadata>>> tsDeviceSeriesMetadataMap
= new LinkedHashMap<>();
- for (String deviceId : devices) {
- Map<String, List<ChunkMetadata>> seriesMetadataMap = reader
- .readChunkMetadataInDevice(deviceId);
- tsDeviceSeriesMetadataMap.put(deviceId, seriesMetadataMap);
- }
+ List<ChunkGroupMetadata> allChunkGroupMetadata = new ArrayList<>();
+ List<Pair<Long, Long>> versionInfo = new ArrayList<>();
+ reader.selfCheck(null, allChunkGroupMetadata, versionInfo, false);
// begin print
StringBuilder str1 = new StringBuilder();
@@ -81,30 +82,28 @@ public class TsFileSketchTool {
String.format("%20d", TSFileConfig.MAGIC_STRING.getBytes().length)
+ "|\t[version number] "
+ reader.readVersionNumber());
- // device begins
- for (Entry<String, Map<String, List<ChunkMetadata>>> entry :
tsDeviceSeriesMetadataMap
- .entrySet()) {
- printlnBoth(pw, str1.toString() + "\t[Chunks] of " + entry.getKey() +
- ", num of Chunks:" + entry.getValue().size());
+ // ChunkGroup begins
+ for (int i = 0; i < allChunkGroupMetadata.size(); i++) {
+ ChunkGroupMetadata chunkGroupMetadata = allChunkGroupMetadata.get(i);
+ printlnBoth(pw, str1.toString() + "\t[Chunk Group] of " +
chunkGroupMetadata.getDevice() +
+ ", num of Chunks:" +
chunkGroupMetadata.getChunkMetadataList().size());
// chunk begins
long chunkEndPos = 0;
- for (Entry<String, List<ChunkMetadata>> seriesMetadata :
entry.getValue().entrySet()) {
- for (ChunkMetadata chunkMetaData : seriesMetadata.getValue()) {
- printlnBoth(pw,
- String.format("%20d", chunkMetaData.getOffsetOfChunkHeader())
+ "|\t[Chunk] of "
- + chunkMetaData.getMeasurementUid() + ", numOfPoints:" +
chunkMetaData
- .getNumOfPoints() + ", time range:[" +
chunkMetaData.getStartTime() + ","
- + chunkMetaData.getEndTime() + "], tsDataType:" +
chunkMetaData.getDataType()
- + ", \n" + String.format("%20s", "") + " \t" +
chunkMetaData.getStatistics());
- printlnBoth(pw, String.format("%20s", "") + "|\t\t[marker] 1");
- printlnBoth(pw, String.format("%20s", "") + "|\t\t[ChunkHeader]");
- Chunk chunk = reader.readMemChunk(chunkMetaData);
- printlnBoth(pw,
- String.format("%20s", "") + "|\t\t" +
chunk.getHeader().getNumOfPages() + " pages");
- chunkEndPos =
- chunkMetaData.getOffsetOfChunkHeader() +
chunk.getHeader().getSerializedSize()
- + chunk.getHeader().getDataSize();
- }
+ for (ChunkMetadata chunkMetadata :
chunkGroupMetadata.getChunkMetadataList()) {
+ printlnBoth(pw,
+ String.format("%20d", chunkMetadata.getOffsetOfChunkHeader()) +
"|\t[Chunk] of "
+ + chunkMetadata.getMeasurementUid() + ", numOfPoints:" +
chunkMetadata
+ .getNumOfPoints() + ", time range:[" +
chunkMetadata.getStartTime() + ","
+ + chunkMetadata.getEndTime() + "], tsDataType:" +
chunkMetadata.getDataType()
+ + ", \n" + String.format("%20s", "") + " \t" +
chunkMetadata.getStatistics());
+ printlnBoth(pw, String.format("%20s", "") + "|\t\t[marker] 1");
+ printlnBoth(pw, String.format("%20s", "") + "|\t\t[ChunkHeader]");
+ Chunk chunk = reader.readMemChunk(chunkMetadata);
+ printlnBoth(pw,
+ String.format("%20s", "") + "|\t\t" +
chunk.getHeader().getNumOfPages() + " pages");
+ chunkEndPos =
+ chunkMetadata.getOffsetOfChunkHeader() +
chunk.getHeader().getSerializedSize()
+ + chunk.getHeader().getDataSize();
}
// chunkGroupFooter begins
printlnBoth(pw, String.format("%20s", chunkEndPos) + "|\t[Chunk Group
Footer]");
@@ -116,8 +115,18 @@ public class TsFileSketchTool {
String.format("%20s", "") + "|\t\t[dataSize] " +
chunkGroupFooter.getDataSize());
printlnBoth(pw, String.format("%20s", "") + "|\t\t[num of chunks] " +
chunkGroupFooter
.getNumberOfChunks());
- printlnBoth(pw, str1.toString() + "\t[Chunks] of "
- + entry.getKey() + " ends");
+ printlnBoth(pw, str1.toString() + "\t[Chunk Group] of "
+ + chunkGroupMetadata.getDevice() + " ends");
+ // versionInfo pair begins
+ printlnBoth(pw,
+ String.format("%20s", chunkEndPos +
chunkGroupFooter.getSerializedSize())
+ + "|\t[Version Info pair]");
+ printlnBoth(pw, String.format("%20s", "") + "|\t\t[marker] 3");
+ printlnBoth(pw,
+ String.format("%20s", "") + "|\t\t[offset] " +
versionInfo.get(i).left);
+ printlnBoth(pw,
+ String.format("%20s", "") + "|\t\t[version] " +
versionInfo.get(i).right);
+
}
// metadata begins
@@ -125,14 +134,36 @@ public class TsFileSketchTool {
printlnBoth(pw, String.format("%20s", reader.getFileMetadataPos() - 1)
+ "|\t[marker] 2");
} else {
printlnBoth(pw,
- String.format("%20s", reader.readFileMetadata().getMetaOffset() +
"|\t[marker] 2"));
+ String.format("%20s", reader.readFileMetadata().getMetaOffset()) +
"|\t[marker] 2");
+ }
+
+ Map<String, List<TimeseriesMetadata>> allTimeseriesMetadata =
reader.getAllTimeseriesMetadata();
+ Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap = new
TreeMap<>();
+
+ for (Map.Entry<String, List<TimeseriesMetadata>> entry :
allTimeseriesMetadata.entrySet()) {
+ String device = entry.getKey();
+ List<TimeseriesMetadata> seriesMetadataList = entry.getValue();
+ for (TimeseriesMetadata seriesMetadata : seriesMetadataList) {
+
timeseriesMetadataMap.put(seriesMetadata.getOffsetOfChunkMetaDataList(),
+ new Pair<>(new Path(device, seriesMetadata.getMeasurementId()),
seriesMetadata));
+
+ }
+ }
+ for (Map.Entry<Long, Pair<Path, TimeseriesMetadata>> entry :
timeseriesMetadataMap.entrySet()) {
+ printlnBoth(
+ pw, String.format("%20s", entry.getKey())
+ + "|\t[ChunkMetadataList] of " + entry.getValue().left
+ + ", tsDataType:" + entry.getValue().right.getTSDataType());
+ printlnBoth(pw,
+ String.format("%20s", "") + "|\t[" +
entry.getValue().right.getStatistics() + "] ");
}
+
for (MetadataIndexEntry metadataIndex :
tsFileMetaData.getMetadataIndex().getChildren()) {
printlnBoth(pw, String.format("%20s", metadataIndex.getOffset())
+ "|\t[MetadataIndex] of " + metadataIndex.getName());
}
- printlnBoth(pw, String.format("%20s", reader.getFileMetadataPos()) +
"|\t[TsFileMetaData]");
+ printlnBoth(pw, String.format("%20s", reader.getFileMetadataPos()) +
"|\t[TsFileMetadata]");
printlnBoth(pw, String.format("%20s", "") + "|\t\t[num of devices] " +
tsFileMetaData
.getMetadataIndex().getChildren().size());
printlnBoth(pw,
@@ -160,7 +191,7 @@ public class TsFileSketchTool {
printlnBoth(pw,
String.format("%20s", (reader.getFileMetadataPos() +
reader.getFileMetadataSize()))
- + "|\t[TsFileMetaDataSize] " + reader.getFileMetadataSize());
+ + "|\t[TsFileMetadataSize] " + reader.getFileMetadataSize());
printlnBoth(pw,
String.format("%20s", reader.getFileMetadataPos() +
reader.getFileMetadataSize() + 4)
@@ -173,6 +204,7 @@ public class TsFileSketchTool {
printlnBoth(pw,
"---------------------------------- TsFile Sketch End
----------------------------------");
+ reader.close();
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 56edfc3..98d2173 100644
---
a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++
b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -156,7 +156,7 @@ public class TsFileRecoverPerformer {
private void recoverResourceFromReader() throws IOException {
try (TsFileSequenceReader reader =
- new TsFileSequenceReader(resource.getFile().getAbsolutePath(), false))
{
+ new TsFileSequenceReader(resource.getFile().getAbsolutePath(), true)) {
for (Entry<String, List<TimeseriesMetadata>> entry :
reader.getAllTimeseriesMetadata()
.entrySet()) {
for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) {
diff --git
a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
new file mode 100644
index 0000000..a881e01
--- /dev/null
+++
b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.writelog.recover;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collections;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter;
+import org.apache.iotdb.db.constant.TestConstant;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.version.VersionController;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.StorageGroupProcessorException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
+import org.apache.iotdb.db.writelog.node.WriteLogNode;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class RecoverResourceFromReaderTest {
+
+ private File tsF;
+ private TsFileWriter writer;
+ private WriteLogNode node;
+ private String logNodePrefix =
TestConstant.OUTPUT_DATA_DIR.concat("testNode/0");
+ private Schema schema;
+ private TsFileResource resource;
+ private VersionController versionController = new VersionController() {
+ private int i;
+
+ @Override
+ public long nextVersion() {
+ return ++i;
+ }
+
+ @Override
+ public long currVersion() {
+ return i;
+ }
+ };
+
+ @Before
+ public void setup() throws IOException, WriteProcessException,
MetadataException {
+ EnvironmentUtils.envSetUp();
+ tsF = SystemFileFactory.INSTANCE.getFile(logNodePrefix, "1-1-1.tsfile");
+ tsF.getParentFile().mkdirs();
+
+ schema = new Schema();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ Path path = new Path(("root.sg.device" + i), ("sensor" + j));
+ MeasurementSchema measurementSchema = new MeasurementSchema("sensor" +
j, TSDataType.INT64, TSEncoding.PLAIN);
+ schema.registerTimeseries(path, measurementSchema);
+ MManager.getInstance().createTimeseries(path.getFullPath(),
measurementSchema.getType(),
+ measurementSchema.getEncodingType(),
measurementSchema.getCompressor(),
+ measurementSchema.getProps());
+ }
+ }
+ schema.registerTimeseries(new Path(("root.sg.device99"), ("sensor4")),
+ new MeasurementSchema("sensor4", TSDataType.INT64, TSEncoding.PLAIN));
+ MManager.getInstance()
+ .createTimeseries("root.sg.device99.sensor4", TSDataType.INT64,
TSEncoding.PLAIN,
+ TSFileDescriptor.getInstance().getConfig().getCompressor(),
Collections.emptyMap());
+ schema.registerTimeseries(new Path(("root.sg.device99"), ("sensor2")),
+ new MeasurementSchema("sensor2", TSDataType.INT64, TSEncoding.PLAIN));
+ MManager.getInstance()
+ .createTimeseries("root.sg.device99.sensor2", TSDataType.INT64,
TSEncoding.PLAIN,
+ TSFileDescriptor.getInstance().getConfig().getCompressor(),
Collections.emptyMap());
+ schema.registerTimeseries(new Path(("root.sg.device99"), ("sensor1")),
+ new MeasurementSchema("sensor1", TSDataType.INT64, TSEncoding.PLAIN));
+ MManager.getInstance()
+ .createTimeseries("root.sg.device99.sensor1", TSDataType.INT64,
TSEncoding.PLAIN,
+ TSFileDescriptor.getInstance().getConfig().getCompressor(),
Collections.emptyMap());
+ writer = new TsFileWriter(tsF, schema);
+
+ TSRecord tsRecord = new TSRecord(100, "root.sg.device99");
+ tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor4",
String.valueOf(0)));
+ writer.write(tsRecord);
+ tsRecord = new TSRecord(2, "root.sg.device99");
+ tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor1",
String.valueOf(0)));
+ writer.write(tsRecord);
+
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ tsRecord = new TSRecord(i, "root.sg.device" + j);
+ for (int k = 0; k < 10; k++) {
+ tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor"
+ k,
+ String.valueOf(k)));
+ }
+ writer.write(tsRecord);
+ }
+ }
+
+ writer.flushAllChunkGroups();
+ writer.getIOWriter().close();
+
+ node = MultiFileLogNodeManager.getInstance().getNode(logNodePrefix +
tsF.getName());
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ String[] measurements = new String[10];
+ String[] values = new String[10];
+ for (int k = 0; k < 10; k++) {
+ measurements[k] = "sensor" + k;
+ values[k] = String.valueOf(k + 10);
+ }
+ InsertPlan insertPlan = new InsertPlan("root.sg.device" + j, i,
measurements, values);
+ node.write(insertPlan);
+ }
+ node.notifyStartFlush();
+ }
+ InsertPlan insertPlan = new InsertPlan("root.sg.device99", 1, "sensor4",
"4");
+ node.write(insertPlan);
+ insertPlan = new InsertPlan("root.sg.device99", 300, "sensor2", "2");
+ node.write(insertPlan);
+ node.close();
+
+ resource = new TsFileResource(tsF);
+ }
+
+ @After
+ public void tearDown() throws IOException, StorageEngineException {
+ EnvironmentUtils.cleanEnv();
+ FileUtils.deleteDirectory(tsF.getParentFile());
+ resource.close();
+ node.delete();
+ }
+
+ @Test
+ public void testResourceRecovery() throws StorageGroupProcessorException,
IOException {
+ // write a broken resourceFile
+ File resourceFile = FSFactoryProducer.getFSFactory()
+ .getFile(resource.getFile() + TsFileResource.RESOURCE_SUFFIX);
+ FileUtils.deleteQuietly(resourceFile);
+ try (OutputStream outputStream = FSFactoryProducer.getFSFactory()
+ .getBufferedOutputStream(resourceFile.getPath())) {
+ ReadWriteIOUtils.write(123, outputStream);
+ }
+
+ TsFileRecoverPerformer performer = new
TsFileRecoverPerformer(logNodePrefix,
+ versionController, resource, true, false);
+
ActiveTimeSeriesCounter.getInstance().init(resource.getFile().getParentFile().getParentFile().getName());
+ performer.recover();
+ assertEquals(1, (long) resource.getStartTimeMap().get("root.sg.device99"));
+ assertEquals(300, (long) resource.getEndTimeMap().get("root.sg.device99"));
+ for (int i = 0; i < 10; i++) {
+ assertEquals(0, (long) resource.getStartTimeMap().get("root.sg.device" +
i));
+ assertEquals(9, (long) resource.getEndTimeMap().get("root.sg.device" +
i));
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
index 40173b5..6ddd7b7 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
@@ -202,7 +202,7 @@ public class BinaryStatistics extends Statistics<Binary> {
@Override
public String toString() {
- return super.toString() + " [fistValue:" + firstValue + ",lastValue:" +
lastValue + "]";
+ return super.toString() + " [firstValue:" + firstValue + ",lastValue:" +
lastValue + "]";
}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 0bd8e92..00553b9 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -528,14 +528,16 @@ public class TsFileSequenceReader implements
AutoCloseable {
*
* @param metadataIndex MetadataIndexEntry
* @param buffer byte buffer
+ * @param deviceId String
* @param timeseriesMetadataMap map: deviceId -> timeseriesMetadata list
*/
private void generateMetadataIndex(MetadataIndexEntry metadataIndex,
ByteBuffer buffer,
- Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap) throws
IOException {
+ String deviceId, Map<String, List<TimeseriesMetadata>>
timeseriesMetadataMap) throws IOException {
switch (metadataIndex.getChildNodeType()) {
case INTERNAL_DEVICE:
case LEAF_DEVICE:
case INTERNAL_MEASUREMENT:
+ deviceId = metadataIndex.getName();
MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer);
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
@@ -545,12 +547,11 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
ByteBuffer nextBuffer =
readData(metadataIndexNode.getChildren().get(i).getOffset(),
endOffset);
- generateMetadataIndex(metadataIndexNode.getChildren().get(i),
nextBuffer,
+ generateMetadataIndex(metadataIndexNode.getChildren().get(i),
nextBuffer, deviceId,
timeseriesMetadataMap);
}
break;
case LEAF_MEASUREMENT:
- String deviceId = metadataIndex.getName();
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
while (buffer.hasRemaining()) {
timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer));
@@ -577,7 +578,7 @@ public class TsFileSequenceReader implements AutoCloseable {
endOffset = metadataIndexEntryList.get(i + 1).getOffset();
}
ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset);
- generateMetadataIndex(metadataIndexEntry, buffer, timeseriesMetadataMap);
+ generateMetadataIndex(metadataIndexEntry, buffer, null,
timeseriesMetadataMap);
}
return timeseriesMetadataMap;
}
@@ -588,7 +589,7 @@ public class TsFileSequenceReader implements AutoCloseable {
metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE);
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(),
metadataIndexPair.right);
Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new
TreeMap<>();
- generateMetadataIndex(metadataIndexPair.left, buffer,
timeseriesMetadataMap);
+ generateMetadataIndex(metadataIndexPair.left, buffer, device,
timeseriesMetadataMap);
List<TimeseriesMetadata> deviceTimeseriesMetadata = new ArrayList<>();
for (List<TimeseriesMetadata> timeseriesMetadataList :
timeseriesMetadataMap.values()) {
deviceTimeseriesMetadata.addAll(timeseriesMetadataList);
@@ -816,8 +817,9 @@ public class TsFileSequenceReader implements AutoCloseable {
*
* @param newSchema the schema on each time series in the file
* @param chunkGroupMetadataList ChunkGroupMetadata List
- * @param fastFinish if true and the file is complete, then newSchema and
newMetaData parameter
- * will be not modified.
+ * @param versionInfo version pair List
+ * @param fastFinish if true and the file is complete, then newSchema and
chunkGroupMetadataList
+ * parameter will be not modified.
* @return the position of the file that is fine. All data after the
position in the file should
* be truncated.
*/
