This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch NewTsFileV2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/NewTsFileV2 by this push:
new c9de562 [To NewTsFileV2] Add tests for
getMeasurementChunkMetadataListMapIterator (#2542)
c9de562 is described below
commit c9de5620fdcc40ec6e14cdc1e93a309b6112a6a6
Author: Steve Yurong Su <[email protected]>
AuthorDate: Thu Jan 21 21:39:00 2021 +0800
[To NewTsFileV2] Add tests for getMeasurementChunkMetadataListMapIterator
(#2542)
Add tests for getMeasurementChunkMetadataListMapIterator
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 30 ++--
...easurementChunkMetadataListMapIteratorTest.java | 153 +++++++++++++++++++++
2 files changed, 174 insertions(+), 9 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index eb9b16d..b70d3a6 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1138,6 +1138,19 @@ public class TsFileSequenceReader implements
AutoCloseable {
return result;
}
+ public Map<String, List<String>> getDeviceMeasurementsMap() throws
IOException {
+ Map<String, List<String>> result = new HashMap<>();
+ for (String device : getAllDevices()) {
+ Map<String, TimeseriesMetadata> timeseriesMetadataMap =
readDeviceMetadata(device);
+ for (TimeseriesMetadata timeseriesMetadata :
timeseriesMetadataMap.values()) {
+ result
+ .computeIfAbsent(device, d -> new ArrayList<>())
+ .add(timeseriesMetadata.getMeasurementId());
+ }
+ }
+ return result;
+ }
+
/**
* get device names which has valid chunks in [start, end)
*
@@ -1220,8 +1233,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
Queue<Pair<Long, Long>> queue = new LinkedList<>();
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(),
metadataIndexPair.right);
- collectEachLeafMeasurementNodeOffsetRange(buffer,
MetadataIndexNodeType.INTERNAL_MEASUREMENT,
- queue);
+ collectEachLeafMeasurementNodeOffsetRange(buffer, queue);
return new Iterator<Map<String, List<ChunkMetadata>>>() {
@@ -1237,7 +1249,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
try {
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
ByteBuffer nextBuffer = readData(startEndPair.left,
startEndPair.right);
- while (buffer.hasRemaining()) {
+ while (nextBuffer.hasRemaining()) {
timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(nextBuffer,
true));
}
for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList)
{
@@ -1255,10 +1267,10 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
private void collectEachLeafMeasurementNodeOffsetRange(ByteBuffer buffer,
- MetadataIndexNodeType type, Queue<Pair<Long, Long>> queue)
- throws IOException {
+ Queue<Pair<Long, Long>> queue) throws IOException {
try {
- MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer);
+ final MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer);
+ final MetadataIndexNodeType metadataIndexNodeType =
metadataIndexNode.getNodeType();
final int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; ++i) {
long startOffset = metadataIndexNode.getChildren().get(i).getOffset();
@@ -1266,11 +1278,11 @@ public class TsFileSequenceReader implements
AutoCloseable {
if (i != metadataIndexListSize - 1) {
endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
}
- if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ if
(metadataIndexNodeType.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
queue.add(new Pair<>(startOffset, endOffset));
- return;
+ continue;
}
- collectEachLeafMeasurementNodeOffsetRange(readData(startOffset,
endOffset), type, queue);
+ collectEachLeafMeasurementNodeOffsetRange(readData(startOffset,
endOffset), queue);
}
} catch (BufferOverflowException e) {
logger.error("Error occurred while collecting offset ranges of
measurement nodes of file {}",
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/MeasurementChunkMetadataListMapIteratorTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/MeasurementChunkMetadataListMapIteratorTest.java
new file mode 100644
index 0000000..6c8f29f
--- /dev/null
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/MeasurementChunkMetadataListMapIteratorTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.read;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.utils.FileGenerator;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MeasurementChunkMetadataListMapIteratorTest {
+
+ private static final String FILE_PATH = FileGenerator.outputDataFile;
+ private final TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
+
+ private int maxDegreeOfIndexNode;
+
+ @Before
+ public void before() {
+ maxDegreeOfIndexNode = conf.getMaxDegreeOfIndexNode();
+ conf.setMaxDegreeOfIndexNode(3);
+ }
+
+ @After
+ public void after() {
+ FileGenerator.after();
+ conf.setMaxDegreeOfIndexNode(maxDegreeOfIndexNode);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ test(1, 1);
+ }
+
+ @Test
+ public void test1() throws IOException {
+ test(1, 10);
+ }
+
+ @Test
+ public void test2() throws IOException {
+ test(2, 1);
+ }
+
+ @Test
+ public void test3() throws IOException {
+ test(2, 2);
+ }
+
+ @Test
+ public void test4() throws IOException {
+ test(2, 100);
+ }
+
+ @Test
+ public void test5() throws IOException {
+ test(50, 2);
+ }
+
+ @Test
+ public void test6() throws IOException {
+ test(50, 50);
+ }
+
+ @Test
+ public void test7() throws IOException {
+ test(50, 100);
+ }
+
+ public void test(int deviceNum, int measurementNum) throws IOException {
+ FileGenerator.generateFile(10000, deviceNum, measurementNum);
+
+ try (TsFileSequenceReader fileReader = new
TsFileSequenceReader(FILE_PATH)) {
+ Map<String, List<String>> deviceMeasurementListMap =
fileReader.getDeviceMeasurementsMap();
+
+ List<String> devices = fileReader.getAllDevices();
+
+ Map<String, Map<String, List<ChunkMetadata>>>
expectedDeviceMeasurementChunkMetadataListMap = new HashMap<>();
+ for (String device : devices) {
+ for (String measurement : deviceMeasurementListMap.get(device)) {
+ expectedDeviceMeasurementChunkMetadataListMap
+ .computeIfAbsent(device, d -> new HashMap<>())
+ .computeIfAbsent(measurement, m -> new ArrayList<>())
+ .addAll(fileReader.getChunkMetadataList(new Path(device,
measurement)));
+ }
+ }
+
+ for (String device : devices) {
+ Map<String, List<ChunkMetadata>> expected =
expectedDeviceMeasurementChunkMetadataListMap
+ .get(device);
+
+ Map<String, List<ChunkMetadata>> actual = new HashMap<>();
+ Iterator<Map<String, List<ChunkMetadata>>> iterator = fileReader
+ .getMeasurementChunkMetadataListMapIterator(device);
+ while (iterator.hasNext()) {
+ Map<String, List<ChunkMetadata>> next = iterator.next();
+ for (Entry<String, List<ChunkMetadata>> entry : next.entrySet()) {
+ actual
+ .computeIfAbsent(entry.getKey(), m -> new ArrayList<>())
+ .addAll(entry.getValue());
+ }
+ }
+
+ check(expected, actual);
+ }
+ }
+
+ FileGenerator.after();
+ }
+
+ private void check(Map<String, List<ChunkMetadata>> expected,
+ Map<String, List<ChunkMetadata>> actual) {
+ Assert.assertEquals(expected.keySet(), actual.keySet());
+ for (String measurement : expected.keySet()) {
+ List<ChunkMetadata> expectedChunkMetadataList =
expected.get(measurement);
+ List<ChunkMetadata> actualChunkMetadataList = actual.get(measurement);
+ Assert.assertEquals(expectedChunkMetadataList.size(),
actualChunkMetadataList.size());
+ final int size = expectedChunkMetadataList.size();
+ for (int i = 0; i < size; ++i) {
+ Assert.assertEquals(expectedChunkMetadataList.get(i).toString(),
+ actualChunkMetadataList.get(i).toString());
+ }
+ }
+ }
+}
