This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch experimental/index in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8cf5af772b20dda505e954ed6dfbdfb810a2ef04 Author: samperson1997 <[email protected]> AuthorDate: Thu Nov 18 09:30:30 2021 +0800 fix bugs --- example/tsfile/pom.xml | 2 +- .../iotdb/tsfile/test1835/TsFileAggregation.java | 45 ++- .../iotdb/tsfile/test1835/TsFileAggregationV2.java | 47 ++- .../iotdb/tsfile/test1835/TsFileRawRead.java | 47 ++- .../iotdb/tsfile/test1835/TsFileRawReadV2.java | 51 ++- .../iotdb/tsfile/test1835/TsFileSketchToolV2.java | 425 --------------------- .../apache/iotdb/tsfile/test1835/TsFileWrite.java | 67 ++-- .../{TsFileWrite.java => TsFileWriteV2.java} | 69 ++-- .../iotdb/tsfile/read/TsFileSequenceReader.java | 155 ++++---- .../tsfile/read/query/executor/TsFileExecutor.java | 3 +- .../apache/iotdb/tsfile/write/TsFileWriter.java | 2 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 169 ++++---- 12 files changed, 375 insertions(+), 707 deletions(-) diff --git a/example/tsfile/pom.xml b/example/tsfile/pom.xml index ce21451..45a9e33 100644 --- a/example/tsfile/pom.xml +++ b/example/tsfile/pom.xml @@ -51,7 +51,7 @@ <configuration> <archive> <manifest> - <mainClass>org.apache.iotdb.tsfile.test1831.TsFileAggregation</mainClass> + <mainClass>org.apache.iotdb.tsfile.test1835.TsFileAggregationV2</mainClass> </manifest> </archive> <descriptorRefs> diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregation.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregation.java index b4de97d..343d2ce 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregation.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregation.java @@ -24,30 +24,38 @@ import org.apache.iotdb.tsfile.read.common.Path; import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import java.io.IOException; public class TsFileAggregation { - private static final String DEVICE1 = "device_"; - public static int chunkNum; - public static int deviceNum = 1; - public static int sensorNum = 1; - public static int fileNum = 1; + private static final String DEVICE1 = "device_1"; + public static int deviceNum; + public static int sensorNum; + public static int fileNum; public static void main(String[] args) throws IOException { - long costTime = 0L; Options opts = new Options(); - // Option chunkNumOption = - // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c"); - // opts.addOption(chunkNumOption); + Option deviceNumOption = + OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d"); + opts.addOption(deviceNumOption); + Option sensorNumOption = + OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m"); + opts.addOption(sensorNumOption); + Option fileNumOption = + OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f"); + opts.addOption(fileNumOption); BasicParser parser = new BasicParser(); CommandLine cl; try { cl = parser.parse(opts, args); - // chunkNum = Integer.parseInt(cl.getOptionValue("c")); + deviceNum = Integer.parseInt(cl.getOptionValue("d")); + sensorNum = Integer.parseInt(cl.getOptionValue("m")); + fileNum = Integer.parseInt(cl.getOptionValue("f")); } catch (Exception e) { e.printStackTrace(); } @@ -56,22 +64,23 @@ public class TsFileAggregation { for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) { // file path String path = - "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/" + "/data/szs/data/data/sequence/root.sg/1/" + deviceNum - + "/test1.tsfile"; + + "." + + sensorNum + + "/test" + + fileIndex + + ".tsfile"; // aggregation query try (TsFileSequenceReader reader = new TsFileSequenceReader(path)) { Path seriesPath = new Path(DEVICE1, "sensor_1"); - long startTime = System.nanoTime(); TimeseriesMetadata timeseriesMetadata = reader.readTimeseriesMetadata(seriesPath, false); long count = timeseriesMetadata.getStatistics().getCount(); - costTime += (System.nanoTime() - startTime); - System.out.println(count); } } - System.out.println( - "Total raw read cost time: " + (System.nanoTime() - totalStartTime) / 1000_000 + "ms"); - System.out.println("Index area cost time: " + costTime / 1000_000 + "ms"); + long totalTime = (System.nanoTime() - totalStartTime) / 1000_000; + System.out.println("Total raw read cost time: " + totalTime + "ms"); + System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms"); } } diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregationV2.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregationV2.java index 8681849..b9b52bf 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregationV2.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregationV2.java @@ -24,30 +24,38 @@ import org.apache.iotdb.tsfile.read.common.Path; import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import java.io.IOException; public class TsFileAggregationV2 { - private static final String DEVICE1 = "device_"; - public static int chunkNum; - public static int deviceNum = 1; - public static int sensorNum = 1; - public static int fileNum = 1; + private static final String DEVICE1 = "device_1"; + public static int deviceNum; + public static int sensorNum; + public static int fileNum; public static void main(String[] args) throws IOException { - long costTime = 0L; Options opts = new Options(); - // Option chunkNumOption = - // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c"); - // opts.addOption(chunkNumOption); + Option deviceNumOption = + OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d"); + opts.addOption(deviceNumOption); + Option sensorNumOption = + OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m"); + opts.addOption(sensorNumOption); + Option fileNumOption = + OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f"); + opts.addOption(fileNumOption); BasicParser parser = new BasicParser(); CommandLine cl; try { cl = parser.parse(opts, args); - // chunkNum = Integer.parseInt(cl.getOptionValue("c")); + deviceNum = Integer.parseInt(cl.getOptionValue("d")); + sensorNum = Integer.parseInt(cl.getOptionValue("m")); + fileNum = Integer.parseInt(cl.getOptionValue("f")); } catch (Exception e) { e.printStackTrace(); } @@ -56,22 +64,23 @@ public class TsFileAggregationV2 { for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) { // file path String path = - "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/" + "/data/szs/data/data/sequence/root.sg/0/" + deviceNum - + "/test0.tsfile"; + + "." + + sensorNum + + "/test" + + fileIndex + + ".tsfile"; // aggregation query - try (TsFileSequenceReader reader = new TsFileSequenceReader(path)) { + try (TsFileSequenceReader reader = new TsFileSequenceReader(path, false)) { Path seriesPath = new Path(DEVICE1, "sensor_1"); - long startTime = System.nanoTime(); TimeseriesMetadata timeseriesMetadata = reader.readTimeseriesMetadataV4(seriesPath, false); long count = timeseriesMetadata.getStatistics().getCount(); - costTime += (System.nanoTime() - startTime); - System.out.println(count); } } - System.out.println( - "Total raw read cost time: " + (System.nanoTime() - totalStartTime) / 1000_000 + "ms"); - System.out.println("Index area cost time: " + costTime / 1000_000 + "ms"); + long totalTime = (System.nanoTime() - totalStartTime) / 1000_000; + System.out.println("Total raw read cost time: " + totalTime + "ms"); + System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms"); } } diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawRead.java index 11d54b2..b3d1646 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawRead.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawRead.java @@ -26,6 +26,8 @@ import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import java.io.IOException; @@ -33,24 +35,30 @@ import java.util.ArrayList; public class TsFileRawRead { - private static final String DEVICE1 = "device_"; - public static int chunkNum; - public static int deviceNum = 1; - public static int sensorNum = 1; - public static int fileNum = 1; + private static final String DEVICE1 = "device_1"; + public static int deviceNum; + public static int sensorNum; + public static int fileNum; public static void main(String[] args) throws IOException { - long costTime = 0L; Options opts = new Options(); - // Option chunkNumOption = - // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c"); - // opts.addOption(chunkNumOption); + Option deviceNumOption = + OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d"); + opts.addOption(deviceNumOption); + Option sensorNumOption = + OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m"); + opts.addOption(sensorNumOption); + Option fileNumOption = + OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f"); + opts.addOption(fileNumOption); BasicParser parser = new BasicParser(); CommandLine cl; try { cl = parser.parse(opts, args); - // chunkNum = Integer.parseInt(cl.getOptionValue("c")); + deviceNum = Integer.parseInt(cl.getOptionValue("d")); + sensorNum = Integer.parseInt(cl.getOptionValue("m")); + fileNum = Integer.parseInt(cl.getOptionValue("f")); } catch (Exception e) { e.printStackTrace(); } @@ -59,9 +67,13 @@ public class TsFileRawRead { for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) { // file path String path = - "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/" + "/data/szs/data/data/sequence/root.sg/1/" + deviceNum - + "/test1.tsfile"; + + "." + + sensorNum + + "/test" + + fileIndex + + ".tsfile"; // raw data query try (TsFileSequenceReader reader = new TsFileSequenceReader(path); @@ -72,17 +84,14 @@ public class TsFileRawRead { QueryExpression queryExpression = QueryExpression.create(paths, null); - long startTime = System.nanoTime(); QueryDataSet queryDataSet = readTsFile.query(queryExpression); while (queryDataSet.hasNext()) { - System.out.println(queryDataSet.next()); + queryDataSet.next(); } - - costTime += (System.nanoTime() - startTime); } } - System.out.println( - "Total raw read cost time: " + (System.nanoTime() - totalStartTime) / 1000_000 + "ms"); - System.out.println("Index area cost time: " + costTime / 1000_000 + "ms"); + long totalTime = (System.nanoTime() - totalStartTime) / 1000_000; + System.out.println("Total raw read cost time: " + totalTime + "ms"); + System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms"); } } diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawReadV2.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawReadV2.java index 2f898dc..96b2d59 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawReadV2.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawReadV2.java @@ -26,6 +26,8 @@ import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import java.io.IOException; @@ -33,38 +35,48 @@ import java.util.ArrayList; public class TsFileRawReadV2 { - private static final String DEVICE1 = "device_"; - public static int chunkNum; - public static int deviceNum = 1; - public static int sensorNum = 1; - public static int fileNum = 1; + private static final String DEVICE1 = "device_1"; + public static int deviceNum; + public static int sensorNum; + public static int fileNum; public static void main(String[] args) throws IOException { - long costTime = 0L; - long totalStartTime = System.nanoTime(); Options opts = new Options(); - // Option chunkNumOption = - // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c"); - // opts.addOption(chunkNumOption); + Option deviceNumOption = + OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d"); + opts.addOption(deviceNumOption); + Option sensorNumOption = + OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m"); + opts.addOption(sensorNumOption); + Option fileNumOption = + OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f"); + opts.addOption(fileNumOption); BasicParser parser = new BasicParser(); CommandLine cl; try { cl = parser.parse(opts, args); - // chunkNum = Integer.parseInt(cl.getOptionValue("c")); + deviceNum = Integer.parseInt(cl.getOptionValue("d")); + sensorNum = Integer.parseInt(cl.getOptionValue("m")); + fileNum = Integer.parseInt(cl.getOptionValue("f")); } catch (Exception e) { e.printStackTrace(); } + long totalStartTime = System.nanoTime(); for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) { // file path String path = - "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/" + "/data/szs/data/data/sequence/root.sg/0/" + deviceNum - + "/test0.tsfile"; + + "." + + sensorNum + + "/test" + + fileIndex + + ".tsfile"; // raw data query - try (TsFileSequenceReader reader = new TsFileSequenceReader(path); + try (TsFileSequenceReader reader = new TsFileSequenceReader(path, false); ReadOnlyTsFile readTsFile = new ReadOnlyTsFile(reader)) { ArrayList<Path> paths = new ArrayList<>(); @@ -72,17 +84,14 @@ public class TsFileRawReadV2 { QueryExpression queryExpression = QueryExpression.create(paths, null); - long startTime = System.nanoTime(); QueryDataSet queryDataSet = readTsFile.query(queryExpression); while (queryDataSet.hasNext()) { - System.out.println(queryDataSet.next()); + queryDataSet.next(); } - - costTime += (System.nanoTime() - startTime); } } - System.out.println( - "Total raw read cost time: " + (System.nanoTime() - totalStartTime) / 1000_000 + "ms"); - System.out.println("Index area cost time: " + costTime / 1000_000 + "ms"); + long totalTime = (System.nanoTime() - totalStartTime) / 1000_000; + System.out.println("Total raw read cost time: " + totalTime + "ms"); + System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms"); } } diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileSketchToolV2.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileSketchToolV2.java deleted file mode 100644 index c36a085..0000000 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileSketchToolV2.java +++ /dev/null @@ -1,425 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tsfile.test1835; - -import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.iotdb.tsfile.file.MetaMarker; -import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader; -import org.apache.iotdb.tsfile.file.header.PageHeader; -import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata; -import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.MetadataIndexEntry; -import org.apache.iotdb.tsfile.file.metadata.MetadataIndexNode; -import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; -import org.apache.iotdb.tsfile.file.metadata.enums.MetadataIndexNodeType; -import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; -import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.read.common.Chunk; -import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.utils.BloomFilter; -import org.apache.iotdb.tsfile.utils.Pair; - -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -public class TsFileSketchToolV2 { - - private String filename; - private String indexFileName; - private PrintWriter pw; - private TsFileSketchToolReader reader; - private TsFileSketchToolReader indexReader; - private String splitStr; // for split different part of TsFile - - public static void main(String[] args) throws IOException { - String filename = "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/1/test0.tsfile"; - String outFile = "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/1/test0.txt"; - String indexFileName = - "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/1/test0.tsfile.index"; - - new TsFileSketchToolV2(filename, indexFileName, outFile).run(); - } - - /** - * construct TsFileSketchTool - * - * @param filename input file path - * @param indexFileName index file path - * @param outFile output file path - */ - public TsFileSketchToolV2(String filename, String indexFileName, String outFile) { - try { - this.filename = filename; - this.indexFileName = indexFileName; - pw = new PrintWriter(new FileWriter(outFile)); - reader = new TsFileSketchToolReader(filename); - indexReader = new TsFileSketchToolReader(indexFileName); - StringBuilder str1 = new StringBuilder(); - for (int i = 0; i < 21; i++) { - str1.append("|"); - } - splitStr = str1.toString(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** entry of tool */ - public void run() throws IOException { - long length = FSFactoryProducer.getFSFactory().getFile(filename).length(); - printlnBoth( - pw, "-------------------------------- TsFile Sketch --------------------------------"); - printlnBoth(pw, "file path: " + filename); - printlnBoth(pw, "file length: " + length); - - // get metadata information - TsFileMetadata tsFileMetaData = reader.readFileMetadataV2(); - List<ChunkGroupMetadata> allChunkGroupMetadata = new ArrayList<>(); - reader.selfCheck(null, allChunkGroupMetadata, false); - - // print file information - printFileInfo(); - - // print chunk - printChunk(allChunkGroupMetadata); - - // metadata begins - if (tsFileMetaData.getMetadataIndex().getChildren().isEmpty()) { - printlnBoth(pw, String.format("%20s", reader.getFileMetadataPos() - 1) + "|\t[marker] 2"); - } else { - printlnBoth( - pw, String.format("%20s", reader.readFileMetadata().getMetaOffset()) + "|\t[marker] 2"); - } - // get all timeseries index - Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap = - reader.getAllTimeseriesMetadataWithOffset(); - - // print timeseries index - printTimeseriesIndex(timeseriesMetadataMap); - - // print TsFile Metadata - printTsFileMetadata(tsFileMetaData); - - printlnBoth(pw, String.format("%20s", length) + "|\tEND of TsFile"); - printlnBoth( - pw, - "---------------------------- IndexOfTimerseriesIndex Tree -----------------------------"); - // print index tree - MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex(); - TreeMap<Long, MetadataIndexNode> metadataIndexNodeMap = new TreeMap<>(); - List<String> treeOutputStringBuffer = new ArrayList<>(); - loadIndexTree(metadataIndexNode, metadataIndexNodeMap, treeOutputStringBuffer, 0); - - // print IndexOfTimerseriesIndex - printIndexOfTimerseriesIndex(metadataIndexNodeMap); - - for (String str : treeOutputStringBuffer) { - printlnBoth(pw, str); - } - printlnBoth( - pw, - "---------------------------------- TsFile Sketch End ----------------------------------"); - pw.close(); - } - - private void printTsFileMetadata(TsFileMetadata tsFileMetaData) { - try { - printlnBoth(pw, String.format("%20s", reader.getFileMetadataPos()) + "|\t[TsFileMetadata]"); - printlnBoth( - pw, String.format("%20s", "") + "|\t\t[meta offset] " + tsFileMetaData.getMetaOffset()); - // bloom filter - BloomFilter bloomFilter = tsFileMetaData.getBloomFilter(); - printlnBoth( - pw, - String.format("%20s", "") - + "|\t\t[bloom filter bit vector byte array length] " - + bloomFilter.serialize().length); - printlnBoth(pw, String.format("%20s", "") + "|\t\t[bloom filter bit vector byte array] "); - printlnBoth( - pw, - String.format("%20s", "") - + "|\t\t[bloom filter number of bits] " - + bloomFilter.getSize()); - printlnBoth( - pw, - String.format("%20s", "") - + "|\t\t[bloom filter number of hash functions] " - + bloomFilter.getHashFunctionSize()); - - printlnBoth( - pw, - String.format("%20s", (reader.getFileMetadataPos() + reader.getFileMetadataSize())) - + "|\t[TsFileMetadataSize] " - + reader.getFileMetadataSize()); - - printlnBoth( - pw, - String.format("%20s", reader.getFileMetadataPos() + reader.getFileMetadataSize() + 4) - + "|\t[magic tail] " - + reader.readTailMagic()); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private void printIndexOfTimerseriesIndex(TreeMap<Long, MetadataIndexNode> metadataIndexNodeMap) { - for (Map.Entry<Long, MetadataIndexNode> entry : metadataIndexNodeMap.entrySet()) { - printlnBoth( - pw, - String.format("%20s", entry.getKey()) - + "|\t[IndexOfTimerseriesIndex Node] type=" - + entry.getValue().getNodeType()); - for (MetadataIndexEntry metadataIndexEntry : entry.getValue().getChildren()) { - printlnBoth( - pw, - String.format("%20s", "") - + "|\t\t<" - + metadataIndexEntry.getName() - + ", " - + metadataIndexEntry.getOffset() - + ">"); - } - printlnBoth( - pw, - String.format("%20s", "") + "|\t\t<endOffset, " + entry.getValue().getEndOffset() + ">"); - } - } - - private void printFileInfo() { - try { - printlnBoth(pw, ""); - printlnBoth(pw, String.format("%20s", "POSITION") + "|\tCONTENT"); - printlnBoth(pw, String.format("%20s", "--------") + " \t-------"); - printlnBoth(pw, String.format("%20d", 0) + "|\t[magic head] " + reader.readHeadMagic()); - printlnBoth( - pw, - String.format("%20d", TSFileConfig.MAGIC_STRING.getBytes().length) - + "|\t[version number] " - + reader.readVersionNumber()); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private void printChunk(List<ChunkGroupMetadata> allChunkGroupMetadata) { - try { - long nextChunkGroupHeaderPos = - (long) TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES; - // ChunkGroup begins - for (ChunkGroupMetadata chunkGroupMetadata : allChunkGroupMetadata) { - printlnBoth( - pw, - splitStr - + "\t[Chunk Group] of " - + chunkGroupMetadata.getDevice() - + ", num of Chunks:" - + chunkGroupMetadata.getChunkMetadataList().size()); - // chunkGroupHeader begins - printlnBoth(pw, String.format("%20s", nextChunkGroupHeaderPos) + "|\t[Chunk Group Header]"); - ChunkGroupHeader chunkGroupHeader = - reader.readChunkGroupHeader(nextChunkGroupHeaderPos, false); - printlnBoth(pw, String.format("%20s", "") + "|\t\t[marker] 0"); - printlnBoth( - pw, String.format("%20s", "") + "|\t\t[deviceID] " + chunkGroupHeader.getDeviceID()); - // chunk begins - for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) { - Chunk chunk = reader.readMemChunk(chunkMetadata); - printlnBoth( - pw, - String.format("%20d", chunkMetadata.getOffsetOfChunkHeader()) - + "|\t[Chunk] of " - + chunkMetadata.getMeasurementUid() - + ", numOfPoints:" - + chunkMetadata.getNumOfPoints() - + ", time range:[" - + chunkMetadata.getStartTime() - + "," - + chunkMetadata.getEndTime() - + "], tsDataType:" - + chunkMetadata.getDataType() - + ", \n" - + String.format("%20s", "") - + " \t" - + chunkMetadata.getStatistics()); - printlnBoth( - pw, - String.format("%20s", "") - + "|\t\t[chunk header] " - + "marker=" - + chunk.getHeader().getChunkType() - + ", measurementId=" - + chunk.getHeader().getMeasurementID() - + ", dataSize=" - + chunk.getHeader().getDataSize() - + ", serializedSize=" - + chunk.getHeader().getSerializedSize()); - - printlnBoth(pw, String.format("%20s", "") + "|\t\t[chunk] " + chunk.getData()); - PageHeader pageHeader; - if (((byte) (chunk.getHeader().getChunkType() & 0x3F)) - == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) { - pageHeader = PageHeader.deserializeFrom(chunk.getData(), chunkMetadata.getStatistics()); - } else { - pageHeader = - PageHeader.deserializeFrom(chunk.getData(), chunk.getHeader().getDataType()); - } - printlnBoth( - pw, - String.format("%20s", "") - + "|\t\t[page] " - + " CompressedSize:" - + pageHeader.getCompressedSize() - + ", UncompressedSize:" - + pageHeader.getUncompressedSize()); - nextChunkGroupHeaderPos = - chunkMetadata.getOffsetOfChunkHeader() - + chunk.getHeader().getSerializedSize() - + chunk.getHeader().getDataSize(); - } - reader.position(nextChunkGroupHeaderPos); - byte marker = reader.readMarker(); - switch (marker) { - case MetaMarker.CHUNK_GROUP_HEADER: - // do nothing - break; - case MetaMarker.OPERATION_INDEX_RANGE: - // skip the PlanIndex - nextChunkGroupHeaderPos += 16; - break; - } - - printlnBoth( - pw, splitStr + "\t[Chunk Group] of " + chunkGroupMetadata.getDevice() + " ends"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - private void printTimeseriesIndex( - Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap) { - try { - for (Map.Entry<Long, Pair<Path, TimeseriesMetadata>> entry : - timeseriesMetadataMap.entrySet()) { - printlnBoth( - pw, - String.format("%20s", entry.getKey()) - + "|\t[TimeseriesIndex] of " - + entry.getValue().left - + ", tsDataType:" - + entry.getValue().right.getTSDataType()); - for (IChunkMetadata chunkMetadata : - reader.getChunkMetadataListV3(entry.getValue().left, false)) { - printlnBoth( - pw, - String.format("%20s", "") - + "|\t\t[ChunkIndex] " - + chunkMetadata.getMeasurementUid() - + ", offset=" - + chunkMetadata.getOffsetOfChunkHeader()); - } - printlnBoth( - pw, - String.format("%20s", "") + "|\t\t[" + entry.getValue().right.getStatistics() + "] "); - } - printlnBoth(pw, splitStr); - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * load by dfs, and sort by TreeMap - * - * @param metadataIndexNode current node - * @param metadataIndexNodeMap result map, key is offset - * @param treeOutputStringBuffer result list, string is index tree - * @param deep current deep - */ - private void loadIndexTree( - MetadataIndexNode metadataIndexNode, - TreeMap<Long, MetadataIndexNode> metadataIndexNodeMap, - List<String> treeOutputStringBuffer, - int deep) - throws IOException { - StringBuilder tableWriter = new StringBuilder("\t"); - for (int i = 0; i < deep; i++) { - tableWriter.append("\t\t"); - } - treeOutputStringBuffer.add( - tableWriter.toString() + "[MetadataIndex:" + metadataIndexNode.getNodeType() + "]"); - for (int i = 0; i < metadataIndexNode.getChildren().size(); i++) { - MetadataIndexEntry metadataIndexEntry = metadataIndexNode.getChildren().get(i); - - treeOutputStringBuffer.add( - tableWriter.toString() - + "└──────[" - + metadataIndexEntry.getName() - + "," - + metadataIndexEntry.getOffset() - + "]"); - if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { - long endOffset = metadataIndexNode.getEndOffset(); - if (i != metadataIndexNode.getChildren().size() - 1) { - endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset(); - } - MetadataIndexNode subNode = - indexReader.getMetadataIndexNode(metadataIndexEntry.getOffset(), endOffset); - metadataIndexNodeMap.put(metadataIndexEntry.getOffset(), subNode); - loadIndexTree(subNode, metadataIndexNodeMap, treeOutputStringBuffer, deep + 1); - } - } - } - - private void printlnBoth(PrintWriter pw, String str) { - System.out.println(str); - pw.println(str); - } - - private class TsFileSketchToolReader extends TsFileSequenceReader { - public TsFileSketchToolReader(String file) throws IOException { - super(file); - } - - public Map<Long, Pair<Path, TimeseriesMetadata>> getAllTimeseriesMetadataWithOffset() - throws IOException { - Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>(); - - // FIXME - ByteBuffer buffer = readData(0, 0); - while (buffer.hasRemaining()) { - int bufferPos = buffer.position(); - TimeseriesMetadata timeseriesMetaData = TimeseriesMetadata.deserializeFrom(buffer, false); - timeseriesMetadataMap.put( - reader.position() + bufferPos, - new Pair<>(new Path("d1", timeseriesMetaData.getMeasurementId()), timeseriesMetaData)); - } - return timeseriesMetadataMap; - } - } -} diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java index aa087b4..d6c64d5 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java @@ -32,6 +32,8 @@ import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema; import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import java.io.File; @@ -42,21 +44,30 @@ import java.util.Random; * addMeasurement(MeasurementSchema measurementSchema) throws WriteProcessException */ public class TsFileWrite { - public static int deviceNum = 1; - public static int sensorNum = 1; - public static int fileNum = 1; + public static int deviceNum; + public static int sensorNum; + public static int fileNum; + public static int pointNum = 100; public static void main(String[] args) { Options opts = new Options(); - // Option chunkNumOption = - // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c"); - // opts.addOption(chunkNumOption); + Option deviceNumOption = + OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d"); + opts.addOption(deviceNumOption); + Option sensorNumOption = + OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m"); + opts.addOption(sensorNumOption); + Option fileNumOption = + OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f"); + opts.addOption(fileNumOption); BasicParser parser = new BasicParser(); CommandLine cl; try { cl = parser.parse(opts, args); - // chunkNum = Integer.parseInt(cl.getOptionValue("c")); + deviceNum = Integer.parseInt(cl.getOptionValue("d")); + sensorNum = Integer.parseInt(cl.getOptionValue("m")); + fileNum = Integer.parseInt(cl.getOptionValue("f")); } catch (Exception e) { e.printStackTrace(); } @@ -64,10 +75,12 @@ public class TsFileWrite { for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) { try { String path = - "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/" + "/data/szs/data/data/sequence/root.sg/0/" + deviceNum + + "." + + sensorNum + "/test" - + 0 + + fileIndex + ".tsfile"; File f = FSFactoryProducer.getFSFactory().getFile(path); if (f.exists()) { @@ -76,23 +89,29 @@ public class TsFileWrite { try { TsFileWriter tsFileWriter = new TsFileWriter(f); - // 1000 timeseries - for (int i = 1; i <= 1000; i++) { - tsFileWriter.registerTimeseries( - new Path(Constant.DEVICE_PREFIX, Constant.SENSOR_ + i), - new UnaryMeasurementSchema(Constant.SENSOR_ + i, TSDataType.INT64, TSEncoding.RLE)); + for (int i = 1; i <= deviceNum; i++) { + for (int j = 1; j <= sensorNum; j++) { + Path path1 = new Path(Constant.DEVICE_PREFIX + i, Constant.SENSOR_ + j); + tsFileWriter.registerTimeseries( + path1, + new UnaryMeasurementSchema( + Constant.SENSOR_ + j, TSDataType.INT64, TSEncoding.RLE)); + } } // construct TSRecord - for (int i = 1; i <= 100; i++) { - TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX); - for (int t = 1; t <= 1000; t++) { - DataPoint dPoint1 = new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong()); - tsRecord.addTuple(dPoint1); - } - // write TSRecord - tsFileWriter.write(tsRecord); - if (i % 100 == 0) { - tsFileWriter.flushAllChunkGroups(); + for (int j = 1; j <= deviceNum; j++) { + for (int i = 1; i <= pointNum; i++) { + TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + j); + for (int t = 1; t <= sensorNum; t++) { + DataPoint dPoint1 = + new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong()); + tsRecord.addTuple(dPoint1); + } + // write TSRecord + tsFileWriter.write(tsRecord); + if (i % 100 == 0) { + tsFileWriter.flushAllChunkGroups(); + } } } tsFileWriter.close(); diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWriteV2.java similarity index 59% copy from example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java copy to example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWriteV2.java index aa087b4..ffc8cc0 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWriteV2.java @@ -32,6 +32,8 @@ import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema; import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import java.io.File; @@ -41,22 +43,31 @@ import java.util.Random; * An example of writing data with TSRecord to TsFile It uses the interface: public void * addMeasurement(MeasurementSchema measurementSchema) throws WriteProcessException */ -public class TsFileWrite { - public static int deviceNum = 1; - public static int sensorNum = 1; - public static int fileNum = 1; +public class TsFileWriteV2 { + public static int deviceNum; + public static int sensorNum; + public static int fileNum; + public static int pointNum = 100; public static void main(String[] args) { Options opts = new Options(); - // Option chunkNumOption = - // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c"); - // opts.addOption(chunkNumOption); + Option deviceNumOption = + OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d"); + opts.addOption(deviceNumOption); + Option sensorNumOption = + OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m"); + opts.addOption(sensorNumOption); + Option fileNumOption = + OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f"); + opts.addOption(fileNumOption); BasicParser parser = new BasicParser(); CommandLine cl; try { cl = parser.parse(opts, args); - // chunkNum = Integer.parseInt(cl.getOptionValue("c")); + deviceNum = 1000; // Integer.parseInt(cl.getOptionValue("d")); + sensorNum = 10; // Integer.parseInt(cl.getOptionValue("m")); + fileNum = 1; // Integer.parseInt(cl.getOptionValue("f")); } catch (Exception e) { e.printStackTrace(); } @@ -64,10 +75,12 @@ public class TsFileWrite { for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) { try { String path = - "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/" + "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/0/" + deviceNum + + "." + + sensorNum + "/test" - + 0 + + fileIndex + ".tsfile"; File f = FSFactoryProducer.getFSFactory().getFile(path); if (f.exists()) { @@ -76,23 +89,29 @@ public class TsFileWrite { try { TsFileWriter tsFileWriter = new TsFileWriter(f); - // 1000 timeseries - for (int i = 1; i <= 1000; i++) { - tsFileWriter.registerTimeseries( - new Path(Constant.DEVICE_PREFIX, Constant.SENSOR_ + i), - new UnaryMeasurementSchema(Constant.SENSOR_ + i, TSDataType.INT64, TSEncoding.RLE)); + for (int i = 1; i <= deviceNum; i++) { + for (int j = 1; j <= sensorNum; j++) { + Path path1 = new Path(Constant.DEVICE_PREFIX + i, Constant.SENSOR_ + j); + tsFileWriter.registerTimeseries( + path1, + new UnaryMeasurementSchema( + Constant.SENSOR_ + j, TSDataType.INT64, TSEncoding.RLE)); + } } // construct TSRecord - for (int i = 1; i <= 100; i++) { - TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX); - for (int t = 1; t <= 1000; t++) { - DataPoint dPoint1 = new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong()); - tsRecord.addTuple(dPoint1); - } - // write TSRecord - tsFileWriter.write(tsRecord); - if (i % 100 == 0) { - tsFileWriter.flushAllChunkGroups(); + for (int j = 1; j <= deviceNum; j++) { + for (int i = 1; i <= pointNum; i++) { + TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + j); + for (int t = 1; t <= sensorNum; t++) { + DataPoint dPoint1 = + new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong()); + tsRecord.addTuple(dPoint1); + } + // write TSRecord + tsFileWriter.write(tsRecord); + if (i % 100 == 0) { + tsFileWriter.flushAllChunkGroups(); + } } } tsFileWriter.close(); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index bf5856d..712a812 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -103,6 +103,8 @@ public class TsFileSequenceReader implements AutoCloseable { private long minPlanIndex = Long.MAX_VALUE; private long maxPlanIndex = Long.MIN_VALUE; + private long startTime; + /** * Create a file reader of the given file. The reader will read the tail of the file to get the * file metadata size.Then the reader will skip the first @@ -131,14 +133,14 @@ public class TsFileSequenceReader implements AutoCloseable { if (FSFactoryProducer.getFSFactory().getFile(file + ".index").exists()) { metadataIndexInput = FSFactoryProducer.getFileInputFactory().getTsFileInput(file + ".index"); } - try { - if (loadMetadataSize) { - loadMetadataSize(); - } - } catch (Throwable e) { - tsFileInput.close(); - throw e; - } + // try { + // if (loadMetadataSize) { + // loadMetadataSize(); + // } + // } catch (Throwable e) { + // tsFileInput.close(); + // throw e; + // } } // used in merge resource @@ -195,19 +197,19 @@ public class TsFileSequenceReader implements AutoCloseable { public void loadMetadataSize() throws IOException { ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES); - if (readTailMagic().equals(TSFileConfig.MAGIC_STRING)) { - tsFileInput.read( - metadataSize, - tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES); - metadataSize.flip(); - // read file metadata size and position - fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize); - fileMetadataPos = - tsFileInput.size() - - TSFileConfig.MAGIC_STRING.getBytes().length - - Integer.BYTES - - fileMetadataSize; - } + // if (readTailMagic().equals(TSFileConfig.MAGIC_STRING)) { + tsFileInput.read( + metadataSize, + tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES); + metadataSize.flip(); + // read file metadata size and position + fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize); + fileMetadataPos = + tsFileInput.size() + - TSFileConfig.MAGIC_STRING.getBytes().length + - Integer.BYTES + - fileMetadataSize; + // } } public long getFileMetadataPos() { @@ -265,8 +267,25 @@ public class TsFileSequenceReader implements AutoCloseable { public TsFileMetadata readFileMetadata() throws IOException { try { if (tsFileMetaData == null) { + long start = System.nanoTime(); + ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES); + tsFileInput.read( + metadataSize, + tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES); + metadataSize.flip(); + + // read file metadata size and position + fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize); + + fileMetadataPos = + tsFileInput.size() + - TSFileConfig.MAGIC_STRING.getBytes().length + - Integer.BYTES + - fileMetadataSize; + resourceLogger.debug("ReadFileMetadata " + (System.nanoTime() - start) / 1000_000L + " ms"); tsFileMetaData = TsFileMetadata.deserializeFrom(readData(fileMetadataPos, fileMetadataSize)); + resourceLogger.debug("ReadFileMetadata " + (System.nanoTime() - start) / 1000_000L + " ms"); } } catch (BufferOverflowException e) { logger.error("Something error happened while reading file metadata of file {}", file); @@ -278,18 +297,23 @@ public class TsFileSequenceReader implements AutoCloseable { public TsFileMetadata readFileMetadataV2() throws IOException { try { if (tsFileMetaData == null) { - long totalSize = metadataIndexInput.size(); + long start = System.nanoTime(); ByteBuffer rootNodeOffsetBuffer = ByteBuffer.allocate(Long.BYTES); - metadataIndexInput.read(rootNodeOffsetBuffer, totalSize - Long.BYTES); + metadataIndexInput.read(rootNodeOffsetBuffer, metadataIndexInput.size() - Long.BYTES); rootNodeOffsetBuffer.flip(); long rootNodeOffset = ReadWriteIOUtils.readLong(rootNodeOffsetBuffer); + resourceLogger.debug( + "ReadFileMetadataV2 " + (System.nanoTime() - start) / 1000_000L + " ms"); + tsFileMetaData = TsFileMetadataV2.deserializeFrom( readData( rootNodeOffset, FSFactoryProducer.getFSFactory().getFile(this.file + ".index").length(), metadataIndexInput)); + resourceLogger.debug( + "ReadFileMetadataV2 " + (System.nanoTime() - start) / 1000_000L + " ms"); } } catch (BufferOverflowException e) { logger.error("Something error happened while reading file metadata of file {}", file); @@ -443,33 +467,12 @@ public class TsFileSequenceReader implements AutoCloseable { return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null; } - public TimeseriesMetadata readTimeseriesMetadataV3(Path path, boolean ignoreNotExists) - throws IOException { - readFileMetadataV2(); - - List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>(); - ByteBuffer buffer = readData(position(), fileMetadataPos); - while (buffer.hasRemaining()) { - try { - timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true)); - } catch (BufferOverflowException e) { - logger.error( - "Something error happened while deserializing TimeseriesMetadata of file {}", file); - throw e; - } - } - // return null if path does not exist in the TsFile - int searchResult = - binarySearchInTimeseriesMetadataList(timeseriesMetadataList, path.getMeasurement()); - return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null; - } - public TimeseriesMetadata readTimeseriesMetadataV4(Path path, boolean ignoreNotExists) throws IOException { readFileMetadataV2(); MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex(); Pair<MetadataIndexEntry, Long> metadataIndexPair = - getMetadataAndEndOffset(deviceMetadataIndexNode, path.getDevice(), true, true); + getMetadataAndEndOffsetV2(deviceMetadataIndexNode, path.getDevice(), true, true); if (metadataIndexPair == null) { if (ignoreNotExists) { return null; @@ -487,7 +490,7 @@ public class TsFileSequenceReader implements AutoCloseable { throw e; } metadataIndexPair = - getMetadataAndEndOffset(metadataIndexNode, path.getMeasurement(), false, false); + getMetadataAndEndOffsetV2(metadataIndexNode, path.getMeasurement(), false, false); } if (metadataIndexPair == null) { return null; @@ -596,6 +599,7 @@ public class TsFileSequenceReader implements AutoCloseable { public List<TimeseriesMetadata> readTimeseriesMetadata(String device, Set<String> measurements) throws IOException { + long start = System.nanoTime(); readFileMetadata(); MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex(); Pair<MetadataIndexEntry, Long> metadataIndexPair = @@ -649,6 +653,8 @@ public class TsFileSequenceReader implements AutoCloseable { } } if (measurementsHadFound.size() == measurements.size()) { + resourceLogger.debug( + "ReadTimeseriesMetadata " + (System.nanoTime() - start) / 1000_000L + " ms"); return resultTimeseriesMetadataList; } } @@ -721,10 +727,12 @@ public class TsFileSequenceReader implements AutoCloseable { public List<TimeseriesMetadata> readTimeseriesMetadataV3(String device, Set<String> measurements) throws IOException { + long start = System.nanoTime(); + readFileMetadataV2(); MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex(); Pair<MetadataIndexEntry, Long> metadataIndexPair = - getMetadataAndEndOffset(deviceMetadataIndexNode, device, true, false); + getMetadataAndEndOffsetV2(deviceMetadataIndexNode, device, true, false); if (metadataIndexPair == null) { return Collections.emptyList(); } @@ -748,7 +756,7 @@ public class TsFileSequenceReader implements AutoCloseable { throw e; } measurementMetadataIndexPair = - getMetadataAndEndOffset(metadataIndexNode, measurementList.get(i), false, false); + getMetadataAndEndOffsetV2(metadataIndexNode, measurementList.get(i), false, false); } if (measurementMetadataIndexPair == null) { return Collections.emptyList(); @@ -775,6 +783,8 @@ public class TsFileSequenceReader implements AutoCloseable { } } if (measurementsHadFound.size() == measurements.size()) { + resourceLogger.debug( + "ReadTimeseriesMetadataV3 " + (System.nanoTime() - start) / 1000_000L + " ms"); return resultTimeseriesMetadataList; } } @@ -782,19 +792,6 @@ public class TsFileSequenceReader implements AutoCloseable { return resultTimeseriesMetadataList; } - public MetadataIndexNode readMetadataIndex() throws IOException { - long totalSize = metadataIndexInput.size(); - ByteBuffer lastNodeSizeBuffer = ByteBuffer.allocate(Integer.BYTES); - metadataIndexInput.read(lastNodeSizeBuffer, totalSize - Integer.BYTES); - lastNodeSizeBuffer.flip(); - - int lastNodeSize = ReadWriteIOUtils.readInt(lastNodeSizeBuffer); - ByteBuffer lastNode = ByteBuffer.allocate(lastNodeSize); - metadataIndexInput.read(lastNode, totalSize - lastNodeSize - Integer.BYTES); - lastNode.flip(); - return MetadataIndexNode.deserializeFrom(lastNode); - } - protected int binarySearchInTimeseriesMetadataList( List<TimeseriesMetadata> timeseriesMetadataList, String key) { int low = 0; @@ -915,7 +912,7 @@ public class TsFileSequenceReader implements AutoCloseable { .collect(Collectors.toList())); } else { // keep traversing - deviceList.addAll(getAllDevices(node)); + deviceList.addAll(getAllDevicesV2(node)); } } return deviceList; @@ -1127,6 +1124,31 @@ public class TsFileSequenceReader implements AutoCloseable { } } + protected Pair<MetadataIndexEntry, Long> getMetadataAndEndOffsetV2( + MetadataIndexNode metadataIndex, String name, boolean isDeviceLevel, boolean exactSearch) + throws IOException { + try { + // When searching for a device node, return when it is not INTERNAL_DEVICE + // When searching for a measurement node, return when it is not INTERNAL_MEASUREMENT + if ((isDeviceLevel + && !metadataIndex.getNodeType().equals(MetadataIndexNodeType.INTERNAL_DEVICE)) + || (!isDeviceLevel + && !metadataIndex.getNodeType().equals(MetadataIndexNodeType.INTERNAL_MEASUREMENT))) { + return metadataIndex.getChildIndexEntry(name, exactSearch); + } else { + Pair<MetadataIndexEntry, Long> childIndexEntry = + metadataIndex.getChildIndexEntry(name, false); + ByteBuffer buffer = + readData(childIndexEntry.left.getOffset(), childIndexEntry.right, metadataIndexInput); + return getMetadataAndEndOffsetV2( + MetadataIndexNode.deserializeFrom(buffer), name, isDeviceLevel, exactSearch); + } + } catch (BufferOverflowException e) { + logger.error("Something error happened while deserializing MetadataIndex of file {}", file); + throw e; + } + } + /** * read data from current position of the input, and deserialize it to a CHUNK_GROUP_FOOTER. <br> * This method is not threadsafe. @@ -1590,15 +1612,6 @@ public class TsFileSequenceReader implements AutoCloseable { return chunkMetadataList; } - public List<ChunkMetadata> getChunkMetadataListV3(Path path, boolean ignoreNotExists) - throws IOException { - TimeseriesMetadata timeseriesMetaData = readTimeseriesMetadataV3(path, ignoreNotExists); - - List<ChunkMetadata> chunkMetadataList = readChunkMetaDataList(timeseriesMetaData); - chunkMetadataList.sort(Comparator.comparingLong(IChunkMetadata::getStartTime)); - return chunkMetadataList; - } - public List<ChunkMetadata> getChunkMetadataList(Path path) throws IOException { return getChunkMetadataList(path, false); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java index b3b43a2..cd547ca 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java @@ -67,8 +67,7 @@ public class TsFileExecutor implements QueryExecutor { queryExpression.setSelectSeries(filteredSeriesPath); } - // metadataQuerier.loadChunkMetaDatas(queryExpression.getSelectedSeries()); - // metadataQuerier.loadChunkMetaDatasV2(queryExpression.getSelectedSeries()); + // metadataQuerier.loadChunkMetaDatas(queryExpression.getSelectedSeries()); metadataQuerier.loadChunkMetaDatasV3(queryExpression.getSelectedSeries()); if (queryExpression.hasQueryFilter()) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java index 4dd528d..625c38c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java @@ -356,7 +356,7 @@ public class TsFileWriter implements AutoCloseable { // } // // public void closeV2() throws IOException { - LOG.info("start close file"); + LOG.info("start close file IN NEW WAY"); flushAllChunkGroups(); fileWriter.endFileV3(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index bba12dc..ac24f39 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -331,86 +331,93 @@ public class TsFileIOWriter { } canWrite = false; } - - public void endFileV2() throws IOException { - long metaOffset = out.getPosition(); - - // serialize the SEPARATOR of MetaData - ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream()); - - // group ChunkMetadata by series - // only contains ordinary path and time column of vector series - Map<Path, List<IChunkMetadata>> chunkMetadataListMap = new TreeMap<>(); - - // time column -> ChunkMetadataList TreeMap of value columns in vector - Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap = new HashMap<>(); - - for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) { - List<ChunkMetadata> chunkMetadatas = chunkGroupMetadata.getChunkMetadataList(); - int idx = 0; - while (idx < chunkMetadatas.size()) { - IChunkMetadata chunkMetadata = chunkMetadatas.get(idx); - if (chunkMetadata.getMask() == 0) { - Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); - chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata); - idx++; - } else if (chunkMetadata.isTimeColumn()) { - // time column of a vector series - Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); - chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata); - idx++; - Map<Path, List<IChunkMetadata>> chunkMetadataListMapInVector = - vectorToPathsMap.computeIfAbsent(series, key -> new TreeMap<>()); - - // value columns of a vector series - while (idx < chunkMetadatas.size() && chunkMetadatas.get(idx).isValueColumn()) { - chunkMetadata = chunkMetadatas.get(idx); - Path vectorSeries = - new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); - chunkMetadataListMapInVector - .computeIfAbsent(vectorSeries, k -> new ArrayList<>()) - .add(chunkMetadata); - idx++; - } - } - } - } - - MetadataIndexNode metadataIndex = flushMetadataIndexV2(chunkMetadataListMap, vectorToPathsMap); - TsFileMetadata tsFileMetaData = new TsFileMetadata(); - tsFileMetaData.setMetadataIndex(metadataIndex); - tsFileMetaData.setMetaOffset(metaOffset); - - long footerIndex = out.getPosition(); - if (logger.isDebugEnabled()) { - logger.debug("start to flush the footer,file pos:{}", footerIndex); - } - - // write TsFileMetaData - int size = tsFileMetaData.serializeTo(out.wrapAsStream()); - if (logger.isDebugEnabled()) { - logger.debug("finish flushing the footer {}, file pos:{}", tsFileMetaData, out.getPosition()); - } - - // write bloom filter - size += tsFileMetaData.serializeBloomFilter(out.wrapAsStream(), chunkMetadataListMap.keySet()); - if (logger.isDebugEnabled()) { - logger.debug("finish flushing the bloom filter file pos:{}", out.getPosition()); - } - - // write TsFileMetaData size - ReadWriteIOUtils.write(size, out.wrapAsStream()); // write the size of the file metadata. - - // write magic string - out.write(MAGIC_STRING_BYTES); - - // close file - out.close(); - if (resourceLogger.isDebugEnabled() && file != null) { - resourceLogger.debug("{} writer is closed.", file.getName()); - } - canWrite = false; - } + // + // public void endFileV2() throws IOException { + // long metaOffset = out.getPosition(); + // + // // serialize the SEPARATOR of MetaData + // ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream()); + // + // // group ChunkMetadata by series + // // only contains ordinary path and time column of vector series + // Map<Path, List<IChunkMetadata>> chunkMetadataListMap = new TreeMap<>(); + // + // // time column -> ChunkMetadataList TreeMap of value columns in vector + // Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap = new HashMap<>(); + // + // for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) { + // List<ChunkMetadata> chunkMetadatas = chunkGroupMetadata.getChunkMetadataList(); + // int idx = 0; + // while (idx < chunkMetadatas.size()) { + // IChunkMetadata chunkMetadata = chunkMetadatas.get(idx); + // if (chunkMetadata.getMask() == 0) { + // Path series = new Path(chunkGroupMetadata.getDevice(), + // chunkMetadata.getMeasurementUid()); + // chunkMetadataListMap.computeIfAbsent(series, k -> new + // ArrayList<>()).add(chunkMetadata); + // idx++; + // } else if (chunkMetadata.isTimeColumn()) { + // // time column of a vector series + // Path series = new Path(chunkGroupMetadata.getDevice(), + // chunkMetadata.getMeasurementUid()); + // chunkMetadataListMap.computeIfAbsent(series, k -> new + // ArrayList<>()).add(chunkMetadata); + // idx++; + // Map<Path, List<IChunkMetadata>> chunkMetadataListMapInVector = + // vectorToPathsMap.computeIfAbsent(series, key -> new TreeMap<>()); + // + // // value columns of a vector series + // while (idx < chunkMetadatas.size() && chunkMetadatas.get(idx).isValueColumn()) { + // chunkMetadata = chunkMetadatas.get(idx); + // Path vectorSeries = + // new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); + // chunkMetadataListMapInVector + // .computeIfAbsent(vectorSeries, k -> new ArrayList<>()) + // .add(chunkMetadata); + // idx++; + // } + // } + // } + // } + // + // MetadataIndexNode metadataIndex = flushMetadataIndexV2(chunkMetadataListMap, + // vectorToPathsMap); + // TsFileMetadata tsFileMetaData = new TsFileMetadata(); + // tsFileMetaData.setMetadataIndex(metadataIndex); + // tsFileMetaData.setMetaOffset(metaOffset); + // + // long footerIndex = out.getPosition(); + // if (logger.isDebugEnabled()) { + // logger.debug("start to flush the footer,file pos:{}", footerIndex); + // } + // + // // write TsFileMetaData + // int size = tsFileMetaData.serializeTo(out.wrapAsStream()); + // if (logger.isDebugEnabled()) { + // logger.debug("finish flushing the footer {}, file pos:{}", tsFileMetaData, + // out.getPosition()); + // } + // + // // write bloom filter + // size += tsFileMetaData.serializeBloomFilter(out.wrapAsStream(), + // chunkMetadataListMap.keySet()); + // if (logger.isDebugEnabled()) { + // logger.debug("finish flushing the bloom filter file pos:{}", out.getPosition()); + // } + // + // // write TsFileMetaData size + // ReadWriteIOUtils.write(size, out.wrapAsStream()); // write the size of the file metadata. + // + // // write magic string + // out.write(MAGIC_STRING_BYTES); + // + // // close file + // out.close(); + // if (resourceLogger.isDebugEnabled() && file != null) { + // resourceLogger.debug("{} writer is closed.", file.getName()); + // } + // canWrite = false; + // } public void endFileV3() throws IOException { long metaOffset = out.getPosition(); @@ -497,7 +504,7 @@ public class TsFileIOWriter { // close file out.close(); if (resourceLogger.isDebugEnabled() && file != null) { - resourceLogger.debug("{} writer is closed.", file.getName()); + resourceLogger.debug("{} writer is closed IN NEW WAY.", file.getName()); } canWrite = false; }
