This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch node_type in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 25e7a33dd81b9f8d93fe889f5eed3a33686d448f Author: samperson1997 <[email protected]> AuthorDate: Wed May 13 13:38:44 2020 +0800 Move MetadataIndexNodeType from entry to node --- .../iotdb/flink/tsfile/util/TSFileConfigUtil.java | 2 +- .../util/TSFileConfigUtilCompletenessTest.java | 2 +- .../resources/conf/iotdb-engine.properties | 4 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +-- .../iotdb/tsfile/common/conf/TSFileConfig.java | 12 ++--- .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 +- .../file/metadata/MetadataIndexConstructor.java | 39 ++++++++-------- .../tsfile/file/metadata/MetadataIndexEntry.java | 32 ++----------- .../tsfile/file/metadata/MetadataIndexNode.java | 27 ++++++++--- .../iotdb/tsfile/read/TsFileSequenceReader.java | 53 ++++++++++++---------- .../file/metadata/MetadataIndexNodeTest.java | 14 +++--- .../tsfile/file/metadata/utils/TestHelper.java | 5 +- .../iotdb/tsfile/read/reader/ReaderTest.java | 4 +- 13 files changed, 101 insertions(+), 103 deletions(-) diff --git a/flink-tsfile-connector/src/main/java/org/apache/iotdb/flink/tsfile/util/TSFileConfigUtil.java b/flink-tsfile-connector/src/main/java/org/apache/iotdb/flink/tsfile/util/TSFileConfigUtil.java index d8b72f1..87232de 100644 --- a/flink-tsfile-connector/src/main/java/org/apache/iotdb/flink/tsfile/util/TSFileConfigUtil.java +++ b/flink-tsfile-connector/src/main/java/org/apache/iotdb/flink/tsfile/util/TSFileConfigUtil.java @@ -50,7 +50,7 @@ public class TSFileConfigUtil { globalConfig.setKerberosKeytabFilePath(config.getKerberosKeytabFilePath()); globalConfig.setKerberosPrincipal(config.getKerberosPrincipal()); globalConfig.setMaxNumberOfPointsInPage(config.getMaxNumberOfPointsInPage()); - globalConfig.setMaxDegreeOfIndexNode(config.getMaxDegreeOfIndexNode()); + globalConfig.setDegreeOfIndexNode(config.getDegreeOfIndexNode()); globalConfig.setMaxStringLength(config.getMaxStringLength()); globalConfig.setPageCheckSizeThreshold(config.getPageCheckSizeThreshold()); globalConfig.setPageSizeInByte(config.getPageSizeInByte()); diff --git a/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java b/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java index eb914ff..96d71ee 100644 --- a/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java +++ b/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java @@ -56,7 +56,7 @@ public class TSFileConfigUtilCompletenessTest { "setKerberosKeytabFilePath", "setKerberosPrincipal", "setMaxNumberOfPointsInPage", - "setMaxDegreeOfIndexNode", + "setDegreeOfIndexNode", "setMaxStringLength", "setPageCheckSizeThreshold", "setPageSizeInByte", diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 4df48bb..753e3f9 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -454,8 +454,8 @@ value_encoder=PLAIN # Data compression method, supports UNCOMPRESSED or SNAPPY. Default value is SNAPPY compressor=SNAPPY -# Maximum degree of a metadataIndex node, default value is 1024 -max_degree_of_index_node=1024 +# Degree of a metadataIndex node, default value is 1024 +degree_of_index_node=1024 #################### diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 4389fa8..48ddea6 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -533,9 +533,9 @@ public class IoTDBDescriptor { TSFileDescriptor.getInstance().getConfig().setCompressor(properties .getProperty("compressor", TSFileDescriptor.getInstance().getConfig().getCompressor().toString())); - TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(Integer.parseInt(properties - .getProperty("max_degree_of_index_node", Integer - .toString(TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode())))); + TSFileDescriptor.getInstance().getConfig().setDegreeOfIndexNode(Integer.parseInt(properties + .getProperty("degree_of_index_node", Integer + .toString(TSFileDescriptor.getInstance().getConfig().getDegreeOfIndexNode())))); } public void loadHotModifiedProps() throws QueryProcessException { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index 8f04ad9..29a01d4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -83,9 +83,9 @@ public class TSFileConfig implements Serializable { */ private int maxNumberOfPointsInPage = 1024 * 1024; /** - * The maximum degree of a metadataIndex node, default value is 1024 + * The degree of a metadataIndex node, default value is 1024 */ - private int maxDegreeOfIndexNode = 1024; + private int degreeOfIndexNode = 1024; /** * Data type for input timestamp, TsFile supports INT32 or INT64. */ @@ -233,12 +233,12 @@ public class TSFileConfig implements Serializable { this.maxNumberOfPointsInPage = maxNumberOfPointsInPage; } - public int getMaxDegreeOfIndexNode() { - return maxDegreeOfIndexNode; + public int getDegreeOfIndexNode() { + return degreeOfIndexNode; } - public void setMaxDegreeOfIndexNode(int maxDegreeOfIndexNode) { - this.maxDegreeOfIndexNode = maxDegreeOfIndexNode; + public void setDegreeOfIndexNode(int degreeOfIndexNode) { + this.degreeOfIndexNode = degreeOfIndexNode; } public String getTimeSeriesDataType() { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java index dd79cec..ceb55a4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java @@ -120,8 +120,8 @@ public class TSFileDescriptor { } conf.setMaxNumberOfPointsInPage(Integer.parseInt( properties.getProperty("max_number_of_points_in_page", Integer.toString(conf.getMaxNumberOfPointsInPage())))); - conf.setMaxDegreeOfIndexNode(Integer.parseInt( - properties.getProperty("max_degree_of_index_node", Integer.toString(conf.getMaxDegreeOfIndexNode())))); + conf.setDegreeOfIndexNode(Integer.parseInt( + properties.getProperty("degree_of_index_node", Integer.toString(conf.getDegreeOfIndexNode())))); conf.setTimeSeriesDataType(properties.getProperty("time_series_data_type", conf.getTimeSeriesDataType())); conf.setMaxStringLength( Integer.parseInt(properties.getProperty("max_string_length", Integer.toString(conf.getMaxStringLength())))); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java index 3d72972..3afed1c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java @@ -32,8 +32,8 @@ import org.apache.iotdb.tsfile.write.writer.TsFileOutput; public class MetadataIndexConstructor { - private static final int MAX_DEGREE_OF_INDEX_NODE = TSFileDescriptor.getInstance().getConfig() - .getMaxDegreeOfIndexNode(); + private static final int DEGREE_OF_INDEX_NODE = TSFileDescriptor.getInstance().getConfig() + .getDegreeOfIndexNode(); private MetadataIndexConstructor() { throw new IllegalStateException("Utility class"); @@ -56,18 +56,18 @@ public class MetadataIndexConstructor { } Queue<MetadataIndexNode> measurementMetadataIndexQueue = new ArrayDeque<>(); TimeseriesMetadata timeseriesMetadata; - MetadataIndexNode currentIndexNode = new MetadataIndexNode(); + MetadataIndexNode currentIndexNode = new MetadataIndexNode( + MetadataIndexNodeType.LEAF_MEASUREMENT); for (int i = 0; i < entry.getValue().size(); i++) { timeseriesMetadata = entry.getValue().get(i); // when constructing from leaf node, every "degree number of nodes" are related to an entry - if (i % MAX_DEGREE_OF_INDEX_NODE == 0) { + if (i % DEGREE_OF_INDEX_NODE == 0) { if (currentIndexNode.isFull()) { - addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, - out); - currentIndexNode = new MetadataIndexNode(); + addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out); + currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT); } currentIndexNode.addEntry(new MetadataIndexEntry(timeseriesMetadata.getMeasurementId(), - out.getPosition(), MetadataIndexNodeType.LEAF_MEASUREMENT)); + out.getPosition())); } timeseriesMetadata.serializeTo(out.wrapAsStream()); } @@ -77,11 +77,11 @@ public class MetadataIndexConstructor { } // if not exceed the max child nodes num, ignore the device index and directly point to the measurement - if (deviceMetadataIndexMap.size() <= MAX_DEGREE_OF_INDEX_NODE) { - MetadataIndexNode metadataIndexNode = new MetadataIndexNode(); + if (deviceMetadataIndexMap.size() <= DEGREE_OF_INDEX_NODE) { + MetadataIndexNode metadataIndexNode = new MetadataIndexNode( + MetadataIndexNodeType.INTERNAL_MEASUREMENT); for (Map.Entry<String, MetadataIndexNode> entry : deviceMetadataIndexMap.entrySet()) { - metadataIndexNode.addEntry(new MetadataIndexEntry(entry.getKey(), out.getPosition(), - MetadataIndexNodeType.INTERNAL_MEASUREMENT)); + metadataIndexNode.addEntry(new MetadataIndexEntry(entry.getKey(), out.getPosition())); entry.getValue().serializeTo(out.wrapAsStream()); } metadataIndexNode.setEndOffset(out.getPosition()); @@ -90,17 +90,16 @@ public class MetadataIndexConstructor { // else, build level index for devices Queue<MetadataIndexNode> deviceMetadaIndexQueue = new ArrayDeque<>(); - MetadataIndexNode currentIndexNode = new MetadataIndexNode(); + MetadataIndexNode currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_DEVICE); MetadataIndexNode deviceMetadataIndexNode; for (Map.Entry<String, MetadataIndexNode> entry : deviceMetadataIndexMap.entrySet()) { // when constructing from internal node, each node is related to an entry if (currentIndexNode.isFull()) { addCurrentIndexNodeToQueue(currentIndexNode, deviceMetadaIndexQueue, out); - currentIndexNode = new MetadataIndexNode(); + currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_DEVICE); } - currentIndexNode.addEntry(new MetadataIndexEntry(entry.getKey(), - out.getPosition(), MetadataIndexNodeType.LEAF_DEVICE)); + currentIndexNode.addEntry(new MetadataIndexEntry(entry.getKey(), out.getPosition())); entry.getValue().serializeTo(out.wrapAsStream()); } addCurrentIndexNodeToQueue(currentIndexNode, deviceMetadaIndexQueue, out); @@ -123,21 +122,21 @@ public class MetadataIndexConstructor { TsFileOutput out, MetadataIndexNodeType type) throws IOException { int queueSize = metadataIndexNodeQueue.size(); MetadataIndexNode metadataIndexNode; - MetadataIndexNode currentIndexNode = new MetadataIndexNode(); + MetadataIndexNode currentIndexNode = new MetadataIndexNode(type); while (queueSize != 1) { for (int i = 0; i < queueSize; i++) { metadataIndexNode = metadataIndexNodeQueue.poll(); // when constructing from internal node, each node is related to an entry if (currentIndexNode.isFull()) { addCurrentIndexNodeToQueue(currentIndexNode, metadataIndexNodeQueue, out); - currentIndexNode = new MetadataIndexNode(); + currentIndexNode = new MetadataIndexNode(type); } currentIndexNode.addEntry(new MetadataIndexEntry(metadataIndexNode.peek().getName(), - out.getPosition(), type)); + out.getPosition())); metadataIndexNode.serializeTo(out.wrapAsStream()); } addCurrentIndexNodeToQueue(currentIndexNode, metadataIndexNodeQueue, out); - currentIndexNode = new MetadataIndexNode(); + currentIndexNode = new MetadataIndexNode(type); queueSize = metadataIndexNodeQueue.size(); } return metadataIndexNodeQueue.poll(); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexEntry.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexEntry.java index 6d53878..5325992 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexEntry.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexEntry.java @@ -22,7 +22,6 @@ package org.apache.iotdb.tsfile.file.metadata; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import org.apache.iotdb.tsfile.file.metadata.enums.MetadataIndexNodeType; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; public class MetadataIndexEntry { @@ -30,18 +29,9 @@ public class MetadataIndexEntry { private String name; private long offset; - /** - * type of the child node at offset - */ - private MetadataIndexNodeType childNodeType; - - public MetadataIndexEntry() { - } - - public MetadataIndexEntry(String name, long offset, MetadataIndexNodeType childNodeType) { + public MetadataIndexEntry(String name, long offset) { this.name = name; this.offset = offset; - this.childNodeType = childNodeType; } public String getName() { @@ -52,10 +42,6 @@ public class MetadataIndexEntry { return offset; } - public MetadataIndexNodeType getChildNodeType() { - return childNodeType; - } - public void setName(String name) { this.name = name; } @@ -64,28 +50,20 @@ public class MetadataIndexEntry { this.offset = offset; } - public void setChildNodeType(MetadataIndexNodeType childNodeType) { - this.childNodeType = childNodeType; - } - public String toString() { - return "<" + name + "," + offset + "," + childNodeType + ">"; + return "<" + name + "," + offset + ">"; } public int serializeTo(OutputStream outputStream) throws IOException { int byteLen = 0; byteLen += ReadWriteIOUtils.write(name, outputStream); byteLen += ReadWriteIOUtils.write(offset, outputStream); - byteLen += ReadWriteIOUtils.write(childNodeType.serialize(), outputStream); return byteLen; } public static MetadataIndexEntry deserializeFrom(ByteBuffer buffer) { - MetadataIndexEntry metadataIndex = new MetadataIndexEntry(); - metadataIndex.setName(ReadWriteIOUtils.readString(buffer)); - metadataIndex.setOffset(ReadWriteIOUtils.readLong(buffer)); - metadataIndex - .setChildNodeType(MetadataIndexNodeType.deserialize(ReadWriteIOUtils.readByte(buffer))); - return metadataIndex; + String name = ReadWriteIOUtils.readString(buffer); + long offset = ReadWriteIOUtils.readLong(buffer); + return new MetadataIndexEntry(name, offset); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java index 28fe5df..a436ba1 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNode.java @@ -25,24 +25,33 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.MetadataIndexNodeType; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; public class MetadataIndexNode { - private static final int MAX_DEGREE_OF_INDEX_NODE = TSFileDescriptor.getInstance().getConfig() - .getMaxDegreeOfIndexNode(); + private static final int DEGREE_OF_INDEX_NODE = TSFileDescriptor.getInstance().getConfig() + .getDegreeOfIndexNode(); private List<MetadataIndexEntry> children; private long endOffset; - public MetadataIndexNode() { + /** + * type of the child node at offset + */ + private MetadataIndexNodeType nodeType; + + public MetadataIndexNode(MetadataIndexNodeType nodeType) { children = new ArrayList<>(); endOffset = -1L; + this.nodeType = nodeType; } - public MetadataIndexNode(List<MetadataIndexEntry> children, long endOffset) { + public MetadataIndexNode(List<MetadataIndexEntry> children, long endOffset, + MetadataIndexNodeType nodeType) { this.children = children; this.endOffset = endOffset; + this.nodeType = nodeType; } public List<MetadataIndexEntry> getChildren() { @@ -57,12 +66,16 @@ public class MetadataIndexNode { this.endOffset = endOffset; } + public MetadataIndexNodeType getNodeType() { + return nodeType; + } + public void addEntry(MetadataIndexEntry metadataIndexEntry) { this.children.add(metadataIndexEntry); } boolean isFull() { - return children.size() == MAX_DEGREE_OF_INDEX_NODE; + return children.size() == DEGREE_OF_INDEX_NODE; } MetadataIndexEntry peek() { @@ -79,6 +92,7 @@ public class MetadataIndexNode { byteLen += metadataIndexEntry.serializeTo(outputStream); } byteLen += ReadWriteIOUtils.write(endOffset, outputStream); + byteLen += ReadWriteIOUtils.write(nodeType.serialize(), outputStream); return byteLen; } @@ -89,7 +103,8 @@ public class MetadataIndexNode { children.add(MetadataIndexEntry.deserializeFrom(buffer)); } long offset = ReadWriteIOUtils.readLong(buffer); - return new MetadataIndexNode(children, offset); + MetadataIndexNodeType nodeType = MetadataIndexNodeType.deserialize(ReadWriteIOUtils.readByte(buffer)); + return new MetadataIndexNode(children, offset, nodeType); } public Pair<MetadataIndexEntry, Long> getChildIndexEntry(String key) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 0e23dc2..4b977a4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -312,9 +312,9 @@ public class TsFileSequenceReader implements AutoCloseable { Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset( deviceMetadataIndexNode, path.getDevice(), MetadataIndexNodeType.INTERNAL_DEVICE); ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); - while (!metadataIndexPair.left.getChildNodeType() - .equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { - MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer); + MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode; + if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer); metadataIndexPair = getMetadataAndEndOffset(metadataIndexNode, path.getMeasurement(), MetadataIndexNodeType.INTERNAL_MEASUREMENT); } @@ -336,19 +336,19 @@ public class TsFileSequenceReader implements AutoCloseable { Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset( deviceMetadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE); List<TimeseriesMetadata> resultTimeseriesMetadataList = new ArrayList<>(); - int maxDegreeOfIndexNode = config.getMaxDegreeOfIndexNode(); - if (measurements.size() > maxDegreeOfIndexNode / Math.log(maxDegreeOfIndexNode)) { - traverseAndReadTimeseriesMetadataInOneDevice(resultTimeseriesMetadataList, metadataIndexPair, - measurements); + int degreeOfIndexNode = config.getDegreeOfIndexNode(); + if (measurements.size() > degreeOfIndexNode / Math.log(degreeOfIndexNode)) { + traverseAndReadTimeseriesMetadataInOneDevice(resultTimeseriesMetadataList, + MetadataIndexNodeType.INTERNAL_MEASUREMENT, metadataIndexPair, measurements); return resultTimeseriesMetadataList; } for (String measurement : measurements) { ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); Pair<MetadataIndexEntry, Long> measurementMetadataIndexPair = metadataIndexPair; List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>(); - while (!measurementMetadataIndexPair.left.getChildNodeType() - .equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { - MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer); + MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode; + if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer); measurementMetadataIndexPair = getMetadataAndEndOffset(metadataIndexNode, measurement, MetadataIndexNodeType.INTERNAL_MEASUREMENT); } @@ -367,11 +367,11 @@ public class TsFileSequenceReader implements AutoCloseable { } private void traverseAndReadTimeseriesMetadataInOneDevice( - List<TimeseriesMetadata> timeseriesMetadataList, + List<TimeseriesMetadata> timeseriesMetadataList, MetadataIndexNodeType type, Pair<MetadataIndexEntry, Long> metadataIndexPair, Set<String> measurements) throws IOException { ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); - switch (metadataIndexPair.left.getChildNodeType()) { + switch (type) { case LEAF_DEVICE: case INTERNAL_MEASUREMENT: MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer); @@ -382,6 +382,7 @@ public class TsFileSequenceReader implements AutoCloseable { endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset(); } traverseAndReadTimeseriesMetadataInOneDevice(timeseriesMetadataList, + metadataIndexNode.getNodeType(), new Pair<>(metadataIndexNode.getChildren().get(i), endOffset), measurements); } break; @@ -432,7 +433,7 @@ public class TsFileSequenceReader implements AutoCloseable { int metadataIndexListSize = metadataIndexNode.getChildren().size(); for (int i = 0; i < metadataIndexListSize; i++) { MetadataIndexEntry metadataIndex = metadataIndexNode.getChildren().get(i); - switch (metadataIndex.getChildNodeType()) { + switch (metadataIndexNode.getNodeType()) { case LEAF_MEASUREMENT: case INTERNAL_MEASUREMENT: for (MetadataIndexEntry index : metadataIndexNode.getChildren()) { @@ -519,8 +520,9 @@ public class TsFileSequenceReader implements AutoCloseable { * @param timeseriesMetadataMap map: deviceId -> timeseriesMetadata list */ private void generateMetadataIndex(MetadataIndexEntry metadataIndex, ByteBuffer buffer, - String deviceId, Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap) throws IOException { - switch (metadataIndex.getChildNodeType()) { + String deviceId, MetadataIndexNodeType type, + Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap) throws IOException { + switch (type) { case INTERNAL_DEVICE: case LEAF_DEVICE: case INTERNAL_MEASUREMENT: @@ -535,7 +537,7 @@ public class TsFileSequenceReader implements AutoCloseable { ByteBuffer nextBuffer = readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset); generateMetadataIndex(metadataIndexNode.getChildren().get(i), nextBuffer, deviceId, - timeseriesMetadataMap); + metadataIndexNode.getNodeType(), timeseriesMetadataMap); } break; case LEAF_MEASUREMENT: @@ -544,9 +546,12 @@ public class TsFileSequenceReader implements AutoCloseable { timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer)); } if (timeseriesMetadataMap.containsKey(deviceId)) { - timeseriesMetadataList.addAll(timeseriesMetadataMap.get(deviceId)); + List<TimeseriesMetadata> exist = timeseriesMetadataMap.get(deviceId); + exist.addAll(timeseriesMetadataList); + timeseriesMetadataMap.put(deviceId, exist); + } else { + timeseriesMetadataMap.put(deviceId, timeseriesMetadataList); } - timeseriesMetadataMap.put(deviceId, timeseriesMetadataList); break; } } @@ -556,8 +561,8 @@ public class TsFileSequenceReader implements AutoCloseable { readFileMetadata(); } Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new HashMap<>(); - List<MetadataIndexEntry> metadataIndexEntryList = tsFileMetaData.getMetadataIndex() - .getChildren(); + MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex(); + List<MetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren(); for (int i = 0; i < metadataIndexEntryList.size(); i++) { MetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i); long endOffset = tsFileMetaData.getMetadataIndex().getEndOffset(); @@ -565,7 +570,8 @@ public class TsFileSequenceReader implements AutoCloseable { endOffset = metadataIndexEntryList.get(i + 1).getOffset(); } ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset); - generateMetadataIndex(metadataIndexEntry, buffer, null, timeseriesMetadataMap); + generateMetadataIndex(metadataIndexEntry, buffer, null, + metadataIndexNode.getNodeType(), timeseriesMetadataMap); } return timeseriesMetadataMap; } @@ -576,7 +582,8 @@ public class TsFileSequenceReader implements AutoCloseable { metadataIndexNode, device, MetadataIndexNodeType.INTERNAL_DEVICE); ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>(); - generateMetadataIndex(metadataIndexPair.left, buffer, device, timeseriesMetadataMap); + generateMetadataIndex(metadataIndexPair.left, buffer, device, + MetadataIndexNodeType.INTERNAL_MEASUREMENT, timeseriesMetadataMap); List<TimeseriesMetadata> deviceTimeseriesMetadata = new ArrayList<>(); for (List<TimeseriesMetadata> timeseriesMetadataList : timeseriesMetadataMap.values()) { deviceTimeseriesMetadata.addAll(timeseriesMetadataList); @@ -599,7 +606,7 @@ public class TsFileSequenceReader implements AutoCloseable { private Pair<MetadataIndexEntry, Long> getMetadataAndEndOffset(MetadataIndexNode metadataIndex, String name, MetadataIndexNodeType type) throws IOException { Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex.getChildIndexEntry(name); - if (!childIndexEntry.left.getChildNodeType().equals(type)) { + if (!metadataIndex.getNodeType().equals(type)) { return childIndexEntry; } ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java index 33b7459..eb1ef3a 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexNodeTest.java @@ -30,14 +30,14 @@ public class MetadataIndexNodeTest { @Test public void testBinarySearchInChildren() { List<MetadataIndexEntry> list = new ArrayList<>(); - MetadataIndexNodeType type = MetadataIndexNodeType.LEAF_MEASUREMENT; - list.add(new MetadataIndexEntry("s0", -1L, type)); - list.add(new MetadataIndexEntry("s5", -1L, type)); - list.add(new MetadataIndexEntry("s10", -1L, type)); - list.add(new MetadataIndexEntry("s15", -1L, type)); - list.add(new MetadataIndexEntry("s20", -1L, type)); + list.add(new MetadataIndexEntry("s0", -1L)); + list.add(new MetadataIndexEntry("s5", -1L)); + list.add(new MetadataIndexEntry("s10", -1L)); + list.add(new MetadataIndexEntry("s15", -1L)); + list.add(new MetadataIndexEntry("s20", -1L)); - MetadataIndexNode metadataIndexNode = new MetadataIndexNode(list, -1L); + MetadataIndexNode metadataIndexNode = new MetadataIndexNode(list, -1L, + MetadataIndexNodeType.LEAF_MEASUREMENT); Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0")); Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10")); Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13")); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java index 6611717..89c414d 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java @@ -43,10 +43,9 @@ public class TestHelper { } private static MetadataIndexNode generateMetaDataIndex() { - MetadataIndexNode metaDataIndex = new MetadataIndexNode(); + MetadataIndexNode metaDataIndex = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT); for (int i = 0; i < 5; i++) { - metaDataIndex.addEntry(new MetadataIndexEntry("d" + i, (long) i * 5, - MetadataIndexNodeType.LEAF_MEASUREMENT)); + metaDataIndex.addEntry(new MetadataIndexEntry("d" + i, (long) i * 5)); } return metaDataIndex; } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java index e9f7a5f..5f9a271 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java @@ -51,7 +51,7 @@ public class ReaderTest { @Before public void before() throws IOException { TSFileDescriptor.getInstance().getConfig().setTimeEncoder("TS_2DIFF"); - TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(3); + TSFileDescriptor.getInstance().getConfig().setDegreeOfIndexNode(3); TsFileGeneratorForTest.generateFile(rowCount, 10 * 1024 * 1024, 10000); fileReader = new TsFileSequenceReader(FILE_PATH); metadataQuerierByFile = new MetadataQuerierByFileImpl(fileReader); @@ -60,7 +60,7 @@ public class ReaderTest { @After public void after() throws IOException { fileReader.close(); - TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(1024); + TSFileDescriptor.getInstance().getConfig().setDegreeOfIndexNode(1024); TsFileGeneratorForTest.after(); }
